Add Deadline native queue prototype

.gitignore

@@ -137,6 +137,10 @@ user/
 
 # Deadline specific
 *.deadline
+deadline_native_settings.json
+work/
+scripts/*_spike.py
+scripts/m0_session_plugin/
 
 # OS specific
 .DS_Store
@@ -153,4 +157,4 @@ Thumbs.db
 
 # Logs
 *.log
-logs/ 
+logs/

README.md

@@ -18,6 +18,8 @@ This plugin adds two nodes:
 
 The submitter does not render the workflow locally. It only packages the job and sends it to Deadline.
 
+The **Deadline** sidebar selects a pool, group and priority, with an optional Worker restriction. **Run (Deadline)** submits through ComfyUI's native Run controls and displays farm work in the existing Jobs panel. Accepted runs survive submitter shutdown. See [native queue setup and limits](docs/DEADLINE_NATIVE_QUEUE.md) and [live validation](docs/NATIVE_QUEUE_VALIDATION.md).
+
 ## Install
 
 Clone this into `ComfyUI/custom_nodes`:

__init__.py

@@ -9,7 +9,11 @@ from .deadline_submit import (
     NODE_DISPLAY_NAME_MAPPINGS,
     register_on_prompt_handler,
 )
+from .farm_backend import register_backend
 
 register_on_prompt_handler()
+FARM_BACKEND = register_backend()
 
-__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
+WEB_DIRECTORY = "./web"
+
+__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"]

deadline_submit.py

@@ -7,6 +7,7 @@ prompt, stages referenced input assets, and submits a render job to Deadline.
 
 import copy
 import filecmp
+import hashlib
 import json
 import os
 import re
@@ -96,7 +97,7 @@ class DeadlineCommandHelper:
         return ""
 
     @staticmethod
-    def call_deadline_command(arguments: List[str], hide_window: bool = True) -> str:
+    def call_deadline_command(arguments: List[str], hide_window: bool = True, timeout: float = 60) -> str:
         deadline_command = DeadlineCommandHelper.get_deadline_command()
         if not deadline_command:
             raise RuntimeError("Deadline command not found. Set DEADLINE_PATH or install Deadline Client.")
@@ -119,7 +120,12 @@ class DeadlineCommandHelper:
             startupinfo=startupinfo,
             creationflags=creationflags,
         )
-        stdout, stderr = process.communicate()
+        try:
+            stdout, stderr = process.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            process.communicate()
+            raise TimeoutError("Deadline command timed out; submission outcome may be unconfirmed") from None
         output = stdout.decode(errors="replace") if isinstance(stdout, bytes) else str(stdout)
         errors = stderr.decode(errors="replace") if isinstance(stderr, bytes) else str(stderr)
 
@@ -171,10 +177,13 @@ class WorkflowProcessor:
 
 
 class InputAssetStager:
-    def __init__(self, output_directory: str, job_name: str, submission_id: str):
+    def __init__(self, output_directory: str, job_name: str, submission_id: str, staging_directory: Optional[str] = None, input_directory: Optional[str] = None, native_assets: bool = False):
         self.output_directory = os.path.abspath(output_directory)
         self.job_name = job_name
         self.submission_id = submission_id
+        self.staging_directory = staging_directory
+        self.input_directory = input_directory
+        self.native_assets = native_assets
 
     def stage_referenced_assets(self, prompt: Dict[str, Any]) -> Tuple[str, str, List[Dict[str, Any]]]:
         input_dir = self._get_local_input_directory()
@@ -203,10 +212,29 @@ class InputAssetStager:
                 if os.path.isabs(original_reference)
                 else original_reference
             )
+            if self.native_assets:
+                # Results added back to a workflow already have a deep run/
+                # attempt namespace. Keep retained native inputs shallow so
+                # Deadline's Python can open them on Windows. Content naming
+                # is stable when retry reads the previously staged copy.
+                checksum = hashlib.sha256()
+                with open(source_path, "rb") as handle:
+                    for block in iter(lambda: handle.read(1024 * 1024), b""):
+                        checksum.update(block)
+                destination_rel_path = checksum.hexdigest()[:24] + os.path.splitext(source_path)[1].lower()
             staged_rel_path, destination_path = self._resolve_destination(staging_dir, destination_rel_path, source_path)
             os.makedirs(os.path.dirname(destination_path), exist_ok=True)
             if not os.path.exists(destination_path):
-                shutil.copy2(source_path, destination_path)
+                fd, temporary = tempfile.mkstemp(prefix="stage-", suffix=".tmp", dir=os.path.dirname(destination_path))
+                os.close(fd)
+                try:
+                    shutil.copy2(source_path, temporary)
+                    if not filecmp.cmp(source_path, temporary, shallow=False):
+                        raise RuntimeError(f"Input changed while staging: {source_path}")
+                    os.replace(temporary, destination_path)
+                finally:
+                    if os.path.exists(temporary):
+                        os.unlink(temporary)
             assets.append({
                 "original_relative_path": original_reference.replace("\\", "/"),
                 "staged_relative_path": staged_rel_path.replace("\\", "/"),
@@ -226,6 +254,8 @@ class InputAssetStager:
         return staging_dir, manifest_path, assets
 
     def _get_local_input_directory(self) -> str:
+        if self.input_directory is not None:
+            return os.path.abspath(self.input_directory)
         try:
             import folder_paths
             return os.path.abspath(folder_paths.get_input_directory())
@@ -276,6 +306,9 @@ class InputAssetStager:
 
         stem, extension = os.path.splitext(rel_path)
         staged_rel_path = f"{stem}_{self.submission_id}{extension}"
+        alternative = os.path.join(staging_dir, staged_rel_path)
+        if os.path.exists(alternative) and not filecmp.cmp(source_path, alternative, shallow=False):
+            staged_rel_path = f"{stem}_{self.submission_id}_{uuid.uuid4().hex[:8]}{extension}"
         return staged_rel_path, os.path.abspath(os.path.join(staging_dir, staged_rel_path))
 
     def _resolve_input_file(
@@ -287,7 +320,12 @@ class InputAssetStager:
     ) -> Optional[Tuple[str, str]]:
         clean_value, annotation = self._strip_annotation(value)
         if annotation in {"output", "temp"}:
-            return None
+            if not self.native_assets:
+                return None
+            import folder_paths
+            input_dir = os.path.abspath(folder_paths.get_output_directory() if annotation == "output" else folder_paths.get_temp_directory())
+        if self.native_assets and strict:
+            allow_absolute = True
         if not clean_value:
             return None
         if os.path.splitext(clean_value)[1].lower() not in MEDIA_EXTENSIONS:
@@ -325,6 +363,8 @@ class InputAssetStager:
         return match.group(1).strip().replace("/", os.sep), match.group(2)
 
     def _staging_directory(self) -> str:
+        if self.staging_directory is not None:
+            return os.path.abspath(self.staging_directory)
         parent = os.path.dirname(self.output_directory.rstrip("\\/"))
         return os.path.join(parent, "input")
 

docs/DEADLINE_NATIVE_QUEUE.md

@@ -0,0 +1,9 @@
+# Run workflows on Deadline
+
+Open **Deadline** in the left sidebar. Choose a pool, group and priority; changes save automatically. **More options** contains task timeout and an optional **Only these Workers** restriction. Leave the restriction empty to let Deadline choose.
+
+Click **Run on Deadline**, or select **Run (Deadline)** in the native Run dropdown. Native batch count and seed controls work as usual. Each workflow becomes a Deadline job, and farm results appear in the existing **Jobs** panel. Accepted runs continue after you close ComfyUI.
+
+Use **Stop** on a queued or running farm job. **Stopping…** remains visible until cancellation is confirmed. Deleted Deadline jobs leave the active queue; their snapshots remain available for history and retry. A committed result stays completed even if its Deadline job is subsequently removed.
+
+Warm reuse is an optional farm configuration, not an artist reservation. If the queue reports missing storage or render helpers, see the [administrator guide](DEADLINE_NATIVE_QUEUE_ADMIN.md). Existing Submit to Deadline and DeadlineSeed nodes remain available.

docs/DEADLINE_NATIVE_QUEUE_ADMIN.md

@@ -0,0 +1,74 @@
+# Native queue administration
+
+Deploy this complete custom-node checkout to the ComfyUI installations used by the farm, then deploy its Deadline render plugin:
+
+```powershell
+.\scripts\deploy_deadline_plugin.ps1 -RepositoryPath '\\FILESERVER\DeadlineRepository'
+```
+
+The deploy script backs up the repository plugin. Configure **ComfyUI Installation Paths** in Deadline Monitor as usual. Normal native queue tasks launch an owned ComfyUI with Deadline's GPU affinity, verify outputs, publish the result and stop the process before the task completes. They need no scheduled session owner or additional Worker. Worker accounts need the workflow's models and custom nodes.
+
+Set **Native Queue Shared Storage** (`NativeQueueRoot`) in Deadline's ComfyUI plugin configuration to an off-host UNC folder accessible for read/write by submitters and Workers, for example `\\FILESERVER\ComfyQueue`. A new submitter reads this site setting at startup and initializes the shared protocol. `COMFY_DEADLINE_NATIVE_ROOT` overrides the site root for isolated tests. Existing local `deadline_native_settings.json` takes precedence, so restart after changing site settings or update that file deliberately.
+
+The local file retains host administration fields `enabled`, `root`, `scope` and `plugin_directory`. They are absent from the artist sidebar. `enabled:false` disables submission without disabling ordinary local Run. The durable history scope is retained to keep existing runs accessible; new installations use the real Deadline user. Jobs use `deadlinecommand -GetCurrentUserName` for ownership independently of that history scope. Changing root or scope is blocked while accepted or unconfirmed runs remain active.
+
+Pool, group, priority, task timeout and the optional Worker restriction save from the sidebar. An empty restriction lets Deadline choose. Each workflow is one independent one-task job; native batch count supplies parallelism. Offline or busy destinations queue normally. Explicit invalid restrictions block submission.
+
+## Optional warm sessions
+
+Install a persistent session owner only on a Worker/GPU dedicated to ComfyUI. It is a process cache and does not reserve a Worker for an artist.
+
+```powershell
+.\scripts\install_native_session.ps1 `
+  -Worker GPU-WORKER `
+  -GpuUuid GPU-REPLACE-WITH-ACTUAL-UUID `
+  -ComfyRoot C:\ComfyUI_windows_portable\ComfyUI `
+  -StateRoot C:\ComfyNative\GPU-WORKER `
+  -BrokerPort 8202 -ComfyPort 8198 -IdleSeconds 600 -ExclusiveWorker
+```
+
+Configure `NativeWorkerSessions` in the Deadline plugin as JSON keyed by Worker name, with its loopback `broker` and NVIDIA `gpu_uuid`. A Worker `ComfyNativeSession` extra-info entry is an optional fallback. Ports and state roots must be unique. The limited interactive logon task uses the installing account; that account must remain logged on with NAS/model access. The Windows job object terminates owned GPU children if their owner dies. Borrowed artist processes are never adopted or killed.
+
+Normal task-owned execution stops the child at each task boundary. The persistent-owner installer requires an exclusive ComfyUI GPU for model-cache warmth.
+
+## Rendering, warmth and reservation
+
+Normal rendering pays startup/model loading for each job and releases the GPU when finished. An optional exclusive owner retains its process and model cache until its idle timeout; Deadline still schedules each independent job.
+
+Actual reservation requires a Deadline session job whose held tasks execute the session's runs themselves. Separate render jobs cannot run on a Worker occupied by a holding task. This second execution path is not implemented or advertised in the sidebar. Adding it requires a durable session claim, cancellation, crash recovery and expiry protocol; keeping a process alive is not a substitute.
+
+## Compatibility and rollback
+
+Native adapters are currently guarded for ComfyUI **0.39.0 / frontend 1.53.10**, English UI, Windows portable Python and NVIDIA GPUs. Validation used Deadline **10.4.2.2** and SMB. Install the whole checkout, including `web`, `plugins` and `scripts`; Python wheel assets are not a supported deployment. Restart the backend and refresh the browser after updating.
+
+The deployment copies `ComfyUI.py`, `ComfyUI.param`, `native_queue_task.py`, `native_queue_assignment.py`, `farm_protocol.py` and `farm_session_owner.py`. Production repository deployment is an administrator operation. Isolated tests can use a private `plugin_directory` without updating the production plugin.
+
+To roll back native submission, set `enabled:false` in the local settings and restart. Restore the repository plugin backup if reverting farm helpers. Preserve shared run records for recovery; do not change root/scope while runs are active.
+
+## Results, Stop, and retry
+
+Native Jobs supports viewing/downloads, opening the readable workflow, and Add to current workflow. Validated outputs retain their image/video/audio groups and live under an isolated `deadline-native/<run>/attempts/...` namespace. A rebuildable copy in the submitting ComfyUI's output folder makes native LoadImage/Video/Audio actions work normally. Deleting that local copy is safe; reconciliation restores it from the share. Keep sufficient local disk capacity for this cache.
+
+Stop first records durable cancellation intent, then suspends Deadline scheduling and stops only that attempt's Worker prompt. It becomes terminal after stop is confirmed. Completion and cancellation serialize under the same per-run lock. The supported frontend groups cancelled jobs under its Failed filter; the adapter labels their rows **Cancelled**. Clearing terminal history hides records rather than deleting their snapshots or files.
+
+For optional persistent owners, an externally suspended task can remain in progress while its prompt stop cannot be confirmed. Reconciliation preserves ownership until cancellation is acknowledged; inspect the owner and Deadline reports before administrative recovery.
+
+**Retry on Deadline** appears in a terminal farm job's native context menu. It creates a new run from the stored snapshot and retained inputs, preserving resolved seeds. Editing the current graph does not change that snapshot. Repeating a request after a lost transport response reconciles the same submission key; another deliberate Run or successful Retry creates another run. An uncertain submission is retained for reconciliation and is never blindly resubmitted.
+
+## Storage and boundaries
+
+The share contains a versioned `store.json`, request-key mappings, and `runs/<UUID>` folders. A run has immutable submission/prompt/workflow/input manifests, a Deadline receipt, cancellation/control state, and isolated attempt/result files. Windows byte-range locks and same-directory atomic replacement protect records. Result publication verifies generation, attempt ownership, exact Deadline assignment start ticks, file paths, sizes, and hashes before updating the accepted-result pointer. No SQLite database is placed on SMB.
+
+Windows UNC record reads use brief exclusive, unbuffered opens to prevent a cached SMB handle from reading a replaced record. Artist and Worker accounts need Modify access, including delete/rename permission, on the native queue folder. Deploy the updated `farm_protocol.py` to both ComfyUI and the Deadline plugin together; changing global SMB caching settings is unnecessary.
+
+There is no automatic retention deletion. Back up the share and establish an administrative retention policy for terminal runs; do not remove active records or files used by retained retries. Local session input/output directories likewise need a retention policy after tasks have released their reservations.
+
+Supported media loaders are listed in `deadline_submit.py` (`INPUT_LOADER_FIELDS` / `ABSOLUTE_FILE_LOADER_FIELDS`). Individual image/video/audio files are staged, including native `[output]`/`[temp]` references. Directory loaders, arbitrary custom file dependencies, browser-interactive nodes, dynamic switches, dynamic output prefixes, partial graphs, and workflows containing Submit to Deadline require adaptation or the existing node-based path. Workflows containing Distributed collectors or distributed upscalers must use local Run; native Deadline routing rejects them before browser worker dispatch. Fixed supported switches are resolved by the existing worker preflight. Workers must have matching node/model/environment dependencies; missing dependencies and missing files produce task diagnostics.
+
+Browser Comfy API credentials are removed from shared submission metadata. Provision authenticated service credentials on Workers independently. This extension uses the existing trusted ComfyUI/Deadline/SMB access model; user scope separates history but is not a new authorization system.
+
+## Rollback and verification
+
+Set `enabled:false` in the local settings and restart to stop new farm submissions. Accepted jobs and controls remain durable. Stop optional warm owners only when their Workers are idle, and retain shared records for history and retries. Restore the repository plugin backup after native tasks finish if reverting deployment. Stopping an owner terminates its owned child; borrowed artist processes remain separate.
+
+See [the validation record](NATIVE_QUEUE_VALIDATION.md) for live evidence and remaining validation limits. M0 fixture scripts are opt-in and are not imported by production registration. No changes to ComfyUI core, installed frontend files, or Distributed are required.

docs/DEADLINE_NATIVE_QUEUE_IMPLEMENTATION_PLAN.md

@@ -0,0 +1,324 @@
+# Deadline integration through ComfyUI's native queue
+
+Product correction after Opus 5.5 review (2026-10-07): normal tasks use a task-owned ComfyUI process on existing Workers. Persistent owners are optional. Warm model reuse is validated only on exclusive ComfyUI GPUs; shared mode releases the process at task completion. The sidebar no longer installs Workers, scheduled tasks or plugins. Artist reservation remains a separate, unimplemented execution mode. Earlier checked lifecycle gates below describe exclusive-owner validation, not warm reuse on every Worker.
+
+Implementation handoff, 2026-10-06. The implementation is now present, uncommitted. See [setup](DEADLINE_NATIVE_QUEUE.md) and [validation evidence](NATIVE_QUEUE_VALIDATION.md). The starting-point notes below describe the pre-implementation baseline. Checked items have implementation and automated or live evidence; unchecked items identify remaining rollout validation, as detailed in the validation log.
+
+## 1. Agreed behavior and boundaries
+
+- Add **Run (Deadline)** to the existing Run dropdown. Keep the existing Run control, batch count, workflow serialization, and seed/widget callbacks.
+- Display local and farm runs in **ComfyUI's existing Jobs panel and history**. Include queued/running/completed/failed/cancelled states, progress, previews, outputs, errors, Stop, and retry.
+- A Deadline settings panel may show connection health, configuration, and worker/session information. Do not build a second job list or result browser.
+- Submit complete, immutable snapshots of independent runs. Do not split one workflow across machines.
+- Keep submission, staging, reconciliation, and controls in Python. Deadline owns farm scheduling; browser code must not choose workers or dispatch prompts to them.
+- Accepted runs must continue after the submitting ComfyUI process or computer shuts down. No dedicated graph coordinator machine is required.
+- Reuse warm worker sessions and clean up idle owned sessions. Existing artist-owned ComfyUI sessions must never be terminated by the farm plugin.
+- Make changes inside this plugin and, where necessary, ComfyUI-Deadline-Distributed. Do not edit ComfyUI core, the installed frontend package, or a frontend source checkout. Runtime adapters shipped inside the plugin are allowed, but must be narrow, guarded, and tested.
+- Preserve existing Submit to Deadline/DeadlineSeed nodes, aliases, and the Distributed Collector/Upscale path.
+
+The earlier `work/deadline-farm-concept.html` mockup contains a separate Farm job list. That part is superseded by this plan. It is not an implementation specification.
+
+## 2. Ownership: what “native queue integration” means
+
+The Jobs panel is the shared user interface. It does not have to be the submitting machine's actual execution queue.
+
+1. Normal ComfyUI submission produces a resolved API prompt and readable workflow snapshot.
+2. The plugin submits that snapshot as a Deadline job and returns a stable ComfyUI-compatible run ID.
+3. A Python adapter exposes local job records plus Deadline-backed records through the API already used by the Jobs panel.
+4. Deadline assigns the job to a worker. The worker queues one prompt into **its own** ComfyUI instance.
+5. The worker reports scheduling/progress to Deadline and publishes ComfyUI results and metadata to shared storage.
+6. The submitting plugin reads those records for display and controls. On restart it reconstructs them without needing an old browser connection or in-memory registry.
+
+Never insert a remote run into the submitting instance's `PromptQueue`: its local executor would render it. Do not create two schedulers by maintaining a local pending list that must continuously feed Deadline. Every accepted independent run must already exist in Deadline.
+
+## 3. Verified starting point
+
+Recheck these before implementation; the workspace and running applications can change.
+
+- Main repository: `C:\AI\ComfyUI_windows_portable4\ComfyUI\custom_nodes\ComfyUI-Deadline-Plugin`, commit `e87701a`, package version `1.0.10`.
+- Distributed repository: sibling `ComfyUI-Deadline-Distributed`, commit `62c81b1`.
+- ComfyUI: `0.39.0`, commit `b0b743566f65daafc423b4fea8a2fbda94b3384a`.
+- Installed frontend: `1.53.10`. Source was inspected at matching tag `v1.53.10`, not frontend `main`.
+- Read-only frontend checkout used for research: `C:\Users\DOMINI~1.MIL\AppData\Local\Temp\comfy-deadline-queue-research\frontend-v1.53.10`. It is temporary; fetch the matching tag again if missing.
+- On the last check, a ComfyUI instance answered on `http://127.0.0.1:8188`; `/system_stats` reported Python `3.12.8`, embedded Python, Windows. `/deadline/session` also responded.
+- `deadlinecommand.exe` was available through `DEADLINE_PATH`; its file version was `10.4.2.2`, and `-GetRepositoryPath` returned `\\NAS01\repository`.
+- Supplied PDG reference: `\\10.35.96.10\Repository\custom\plugins\PDGDeadline\PDGDeadline.py`. Verify repository aliases before deploying anything.
+- Existing regression suite: **30 tests passed**. These use stubs and do not prove live Deadline scheduling, cancellation, persistence, or GPU reuse. One test intentionally logs a synthetic failure traceback while passing.
+- No real farm jobs, worker failures, adapter behavior, or persistent process lifecycle were exercised for this plan. No runtime implementation was changed.
+
+Read the applicable `AGENTS.md`, including `ComfyUI\AGENTS.md`. Inspect current diffs before editing. Use the checkout's Python modules explicitly: an older installed `deadline_submit` package was found to shadow this checkout in an earlier test invocation.
+
+### Existing code to reuse
+
+- `deadline_submit.py`: `DeadlineCommandHelper`, `WorkflowProcessor`, `InputAssetStager`, `DeadlineJobSubmitter`, prompt/readable-workflow asset rewrites, existing seed handling.
+- `__init__.py`: current node registration and submit-only prompt handler. The new frontend extension needs `WEB_DIRECTORY`; this repository currently does not export it.
+- `plugins/ComfyUI/ComfyUI.py`: endpoint/GPU identity verification, own-prompt tracking, staging for reused endpoints, worker `/object_info` preflight, fixed-switch handling, metadata, history/error extraction, output existence validation, Deadline progress/failure reporting.
+- `plugins/ComfyUI/ComfyUI.param`: existing installation and worker endpoint settings.
+- `scripts/deploy_deadline_plugin.ps1`: deploys `plugins/ComfyUI` to the Deadline repository; account for any added helper files and worker-side extension installation.
+- Distributed `deadline_worker_registration.py`, `deadline_integration_simple.py`, and `worker_monitor.py`: worker identity, registration/heartbeat, and process monitoring concepts.
+- Distributed `web/main.js`: existing sidebar registration pattern.
+
+Reuse needs adaptation, not wholesale activation: Distributed currently registers against `COMFY_MASTER_*`, keeps claims in memory, and holds registration jobs alive. Its browser scheduling in `web/executionUtils.js` and execution patches for workflow splitting are not the scheduler for this feature. Preserve their existing behavior.
+
+Read-only integration references in the matching frontend source: `src/components/actionbar/ComfyRunButton/ComfyQueueButton.vue`, `src/scripts/app.ts` (`queuePrompt`), `src/scripts/api.ts` (`apiURL`, `queuePrompt`, cancellation), `src/platform/remote/comfyui/jobs/`, `src/stores/queueStore.ts`, `src/stores/executionStore.ts`, `src/stores/jobPreviewStore.ts`, and `src/composables/queue/useJobMenu.ts`. In ComfyUI, inspect `server.py` (Jobs routes, middleware/application setup, `add_routes`, events), `main.py` (startup and `prompt_worker`), and the pure helpers in `comfy_execution/jobs.py`.
+
+## 4. Scheduling decision and persistent session spike
+
+**Default implementation: one independent run = one Deadline job with one task**, grouped under a batch name for a native multi-run submission. Use `BatchCount=1` and `ChunkSize=1` for this path. Retain the existing node-based variation/batch behavior separately. Deadline priorities, pools, groups, limits, and GPU/Worker affinity govern scheduling.
+
+Do not regenerate resolved seeds on the worker for this new path. The snapshot received from normal frontend submission is the run to execute. Explicit retry uses that same snapshot.
+
+Evaluate these options in milestone M0 and record measurements:
+
+| Option | Scheduling and lifetime | Decision |
+| --- | --- | --- |
+| One job/task per run, verified reusable worker-local ComfyUI session | Durable independent scheduling; process reuse can work across jobs | Preferred, subject to lifecycle/ownership tests |
+| Fixed batch job with all tasks defined before submission, Advanced Deadline plugin | Can retain an application between tasks within a job; runs may still execute on different Workers | Optional later optimization; prove per-run Stop/retry and distribution first |
+| Persistent Deadline slot jobs pulling a shared queue | Needs a durable claim/dispatch system and separate resource ownership | Do not use for MVP; adds an unnecessary scheduler unless measured benefits justify it |
+| Append tasks to an active Deadline job | Requires reliable dynamic append under farm load | Excluded from MVP; no assumption of reliability |
+
+PDG's transferable pattern is backend submission of work-item specifications and backend result reporting. Interactive PDG still has a local graph coordinator; Submit as Job moves that graph cook to a farm `hython`. Independent fully submitted ComfyUI runs do not need that graph coordinator. SideFX also documents a dynamic task-append race and a one-job-per-work-item workaround. [SideFX Deadline Scheduler](https://www.sidefx.com/docs/houdini/nodes/top/deadlinescheduler.html)
+
+Deadline documents Advanced plugins retaining applications across tasks. That is not proof of application persistence across separate jobs or sandbox teardown. The existing ComfyUI Deadline plugin is Simple, and its distributed keep-alive loop has no idle-drain protocol. [Deadline application plugins](https://docs.thinkboxsoftware.com/products/deadline/10.4/1_User%20Manual/manual/application-plugins.html)
+
+For cross-job warmth, prefer a verified worker-local session owner/helper which survives the individual render task. It coordinates only that machine's process lifetime, not the farm queue. Prove that Deadline task/sandbox cleanup leaves that session alive, and that crashes and shutdowns cannot leave unmanaged GPU processes behind. Do not rely on a detached child surviving by accident.
+
+Session requirements:
+
+- Identify an owned session by Deadline Worker identity, GPU UUID, ComfyUI root, PID, and session UUID. A port number alone is not identity. One active farm prompt per owned GPU/session in MVP.
+- Track active task/prompt reservations locally; a heartbeat expiring is not permission to kill an actively rendering process.
+- Keep input/output roots stable for a persistent process. Use run/attempt namespaces; do not restart merely to change a per-job `--input-directory` or `--output-directory`.
+- A short configurable idle timeout retains warmth; an owned idle session drains and exits afterward. It stops accepting reservations before shutdown, then rechecks active work.
+- Deadline task completion releases scheduling capacity. A warm process still holding VRAM must not conflict with the next non-Comfy job: start with exclusive Comfy GPU Workers, or prove a worker-local handoff that frees the session before another application uses that GPU.
+- Reused artist endpoints are borrowed. Never kill them or globally interrupt unrelated queued/running prompts. Record that their cleanup policy differs from owned sessions.
+- Shared registration data is discovery/diagnostics, not authority to dispatch farm work. Adapt Distributed registration to a durable/local destination; master workstation callbacks must not be required for accepted runs.
+
+## 5. Durable run, submission, and result protocol
+
+Use Deadline for scheduling and a farm-visible share for immutable payloads and result/control metadata. A submitter-local cache/index is rebuildable. Do not place the only registry or cancellation flag in browser storage or local SQLite. Do not put a concurrently written SQLite database on SMB.
+
+Define and version a small JSON protocol, preferably with one shared implementation usable by the submitter and deployed Deadline plugin:
+
+- `run_id`: canonical lowercase UUID; returned as native `prompt_id` and used as the native job ID. Do not expose a Deadline job ID as a native UUID.
+- `submission_key`: stable request ID for transport retries of the same snapshot; associate it with a payload hash. Same key with different payload must be rejected.
+- `batch_id`, `workflow_id`, submitter/user scope, creation time, readable name, and schema version.
+- Exact API prompt, readable workflow, execution options, staging manifest, snapshot hash, and captured configuration. Do not mutate these after preparation.
+- Deadline job/task IDs in the accepted receipt and job metadata, with a manifest locator. Records must be discoverable after losing the submitting computer.
+- `attempt_id`, generation/fencing number, Deadline Worker identity, session UUID, worker prompt ID, timestamps, and progress sequence.
+- A cancellation intent and a pointer to the single accepted result manifest.
+
+Separate deliberate Run clicks are distinct runs even when their snapshot hashes match. Deduplicate transport retries by request key, not all identical workflows by content hash.
+
+Suggested layout under a configurable **off-host** root, not automatically inside Deadline's plugin directory:
+
+```text
+<farm-root>/runs/<run_id>/
+  submission.json        # immutable snapshot/configuration/hash
+  prompt.json
+  workflow.json
+  inputs/                # or references to existing staged immutable assets
+  input_manifest.json
+  receipt.json           # accepted Deadline job/task mapping
+  control.json           # generation, cancellation, accepted result pointer
+  attempts/<attempt_id>/ # isolated progress, errors, preview, outputs, result.json
+```
+
+Reuse `InputAssetStager` and its existing supported-loader behavior. Refactor only the pieces needed for a per-run directory and a non-node submission entry point. Keep unsupported directory-loader limitations explicit. Preserve API and readable workflow asset rewrites together. Model files/custom nodes remain a worker environment responsibility; record useful environment fingerprints and report missing requirements.
+
+### Submission and restart recovery
+
+1. Validate request shape, settings, and staging accessibility; write the complete payload atomically before submitting anything runnable.
+2. Serialize submissions for the same key. Put the run ID, protocol marker, and manifest locator into Deadline's supported metadata fields.
+3. Prefer initial suspended submission, then persist the Job ID mapping and activate the job. Verify these Deadline commands and state transitions on the installed version before depending on them.
+4. Acknowledge acceptance only after a recoverable mapping exists and activation is confirmed. Record the receipt before sending the HTTP response.
+5. On a submission timeout or lost response, reconcile by the durable key/metadata. Do not blindly call SubmitJob again. Surface an unconfirmed state and prevent a second submission until reconciled.
+6. If the submitter crashes before activation, the run was not yet accepted; another connected plugin instance must be able to recover the suspended job. If it crashes after activation but before replying, discover the running job rather than duplicating it.
+7. A native batch may be accepted partially. Report accepted IDs and failed/unconfirmed members clearly. Do not claim an entire batch is farm-owned while some runs exist only in a browser loop.
+
+### Result publication and retry fencing
+
+Use a tested server-enforced lock for shared `control.json`, plus atomic file replacement on the same share. On Windows SMB, evaluate OS byte-range locking; test from two different hosts, including process death. Atomic rename alone is not a compare-and-swap or fencing mechanism. If the storage cannot provide the required exclusion, stop and report the exact unmet durability requirement before selecting a different design.
+
+1. Verify the task assignment belongs to this worker/attempt. At attempt start, under the run lock, reject cancellation; if a result is already committed, finish/reconcile without rendering it again. Otherwise increment generation and record the new attempt. Do not let a stale callback reclaim ownership merely because it can still reach the share.
+2. Execute into an attempt-specific namespace. Track only the returned worker prompt ID and capture its history, outputs, and errors.
+3. Validate and copy outputs before publication. Write an immutable result manifest with normalized node outputs, paths, sizes/hashes as needed, workflow metadata, and attempt identity.
+4. Under the lock, publish the result pointer only if this attempt still owns the generation and cancellation has not won. Late superseded attempts may retain diagnostic files but cannot become visible results.
+5. Publish the manifest before reporting Deadline task success. If storage is unavailable, do not report successful completion with missing metadata.
+6. Test the “result committed, Deadline acknowledgement lost” case. A retry must find that commit and avoid another visible result; reconciliation must make the final status consistent.
+
+The required guarantee is one accepted visible result per run, not exactly-once execution or arbitrary third-party node side effects. Deadline may retry execution. An explicit user Retry creates a **new run ID**, linked to the original, from its stored snapshot/settings with identical resolved seeds. Automatic infrastructure attempts remain under the original run ID.
+
+## 6. Native Jobs API and frontend integration
+
+Implement a plugin-owned Python Jobs adapter. Preferred seam: install narrow aiohttp middleware during custom-node startup, before the application freezes. The inspected startup order creates `PromptServer`, imports custom nodes, then adds routes/starts the server. This is an internal runtime seam, not a verified public remote-job-provider API.
+
+The inspected frontend reads lists and details through `/api/jobs`, via `api.fetchApi('/jobs...')`. Its schema supports the terminal and active states needed here. Inspect the matching source before shaping responses. [Jobs fetchers](https://github.com/Comfy-Org/ComfyUI_frontend/blob/v1.53.10/src/platform/remote/comfyui/jobs/fetchJobs.ts), [Jobs schema](https://github.com/Comfy-Org/ComfyUI_frontend/blob/v1.53.10/src/platform/remote/comfyui/jobs/jobTypes.ts)
+
+Required behavior:
+
+- `GET /api/jobs`: merge normalized local and farm records, then apply filters, sorting, pagination, and totals to the combined set. Do not concatenate two already paginated pages. Use existing pure normalization helpers where suitable; do not change the underlying executor queue/history.
+- `GET /api/jobs/<run_id>`: return the stored submission workflow, complete normalized outputs, status, timestamps, and structured errors for farm IDs. Local IDs use normal behavior.
+- `GET /api/jobs/<run_id>/assets`: provide native-compatible asset entries where needed by this frontend; use the same accepted manifest as job detail.
+- Native single/batch cancellation endpoints: route farm IDs to the farm controller, local IDs to normal cancellation. Preserve validation, idempotence, and mixed batches.
+- Audit actual consumers of `/queue`, `/history`, `/interrupt`, clear/delete, and their `/api` aliases. Support required compatibility views/controls, without changing the real local queue. Do not register duplicate routes over existing handlers.
+- Clearing pending jobs cancels the addressed scoped runs. Clearing/deleting history hides records according to the chosen user policy; it must not implicitly delete Deadline jobs or shared output files. Retain terminal receipts/results for the configured retention period even if Deadline purges its own finished jobs.
+- Completed media must be available through native view/download URLs. Prefer lazy materialization into a namespaced local output cache where this also makes “Add to workflow” work; otherwise use a plugin-owned safe asset resolver and test every consumer. Never serve an arbitrary supplied filesystem path.
+- Scope records to the current configured user/workspace policy. Do not expose every artist's farm jobs by default. Preserve that identity across restart and document how another authorized workstation can recover the same scope.
+- Keep local operations available if Deadline is unreachable. Cache farm records with a visible stale/connection indicator rather than deleting rows or inventing completion/failure.
+- Backend polling must be batched, bounded, and off the aiohttp event loop. Start with cached state refresh around 2 seconds while active and slower idle polling; measure on the actual farm. Do not spawn one Deadline subprocess per row or browser poll.
+
+Native lifecycle bridging needs its own prototype:
+
+- Send plugin-specific updates keyed by run ID, attempt generation, and sequence; map them into the native job progress/preview state through a small frontend compatibility adapter.
+- Do not indiscriminately replay remote `execution_start`, `executing`, `executed`, or legacy binary previews as if all Workers were one local executor. The inspected frontend has per-job progress maps **and** global active-job/canvas state. Its frame coalescing also needs concurrency testing.
+- Preserve foreground workflow highlighting and local execution while two remote workflows use identical node IDs. Do not attach one worker's preview/error to another run or the currently open unrelated graph.
+- Restore current progress/preview after reconnect from the durable latest state. Live preview loss must not prevent execution or final result publication. Throttle preview files/events and bound retention.
+- Check native active count, workflow filters, and lists with more than 200 active runs; the inspected fetcher requests a maximum active page of 200. Do not silently mistake the loaded page for the farm's full queue.
+
+### Run menu and retry
+
+Add a web extension under this repository and export `WEB_DIRECTORY` from `__init__.py`. Use supported extension settings/commands/sidebar hooks first. The inspected `ComfyQueueButton.vue` constructs its Run mode menu internally; extension menu commands do not automatically add a dropdown item. Its exact dropdown integration must therefore be proved in M0.
+
+- Keep `app.queuePrompt`'s normal widget callbacks and graph serialization. Route the already resolved submission to the plugin backend at a narrow API boundary; preserve all method arguments/options and existing wrapper chains. Do not rebuild the batch seed loop independently.
+- Freeze execution destination/settings for a native finite batch, even if the user changes selection while submission is in progress. Associate submissions explicitly; avoid an unscoped global “next request is farm” flag.
+- Return the native prompt response shape with the canonical run ID and validation errors, so normal job tracking continues.
+- Initial farm support is manual Run plus finite batch count. Existing local Run (On Change)/Run (Instant) retain their behavior. These browser-driven automatic modes must not silently create unlimited farm jobs; farm support for them is a later explicit policy.
+- Handle workflows already containing a Submit to Deadline node without recursively submitting it again. Define and test a single farm submission path; preserve the existing local node-based path.
+- Partial/node execution is outside the first whole-workflow release. Handle it explicitly instead of silently expanding a partial run to a full workflow. Also document Shift+Run/front behavior: Deadline ordering follows its scheduling settings, not a local queue insertion position.
+- The inspected native job menu has no Retry entry. Prove a plugin-added job action in M0. Retry must use the stored snapshot through Python, not reopen a graph and run its widget callbacks again. Do not silently substitute a separate job panel.
+- Prefer supported hooks; if a runtime DOM/UI adapter is necessary, isolate it, identify its version/element assumptions, preserve keyboard/focus behavior, remove observers/listeners on teardown, and detect unsupported frontend versions. Do not patch minified bundles, Vue closures, or stock files.
+
+If the required dropdown/job-action/progress seams cannot be integrated reliably on this frontend, report the exact missing hook and demonstrated failure. Do not quietly change the product to a second Farm jobs UI. A different frontend version or upstream extension hook would be a separate decision.
+
+## 7. Python control and status semantics
+
+Plugin-specific routes, registered without the `/api` prefix because ComfyUI adds aliases:
+
+- `GET /deadline/farm/capabilities`: protocol/version support, available transport, native adapter health, worker-session support, configuration readiness.
+- `GET /deadline/farm/settings` and a validated update route: shared root, pool/group/priority, worker limits/allowlist, timeouts, endpoint/session policy. Reuse native settings UI where practical.
+- `POST /deadline/farm/submit`: snapshot, idempotency key, batch association, captured options/settings; returns a native-compatible accepted run response.
+- `POST /deadline/farm/<run_id>/retry`: creates a new run from the stored snapshot with an idempotent retry request key.
+- Plugin cancellation/reconciliation functions used by native controls; add public routes only if a consumer needs them.
+
+Use `DeadlineCommandHelper` initially, extending it for bounded query/control operations rather than introducing another service. Confirm command syntax and parse structured output on the installed Deadline version. A configured Deadline Web Service can be a later transport; do not add a required coordinator service.
+
+Status policy:
+
+- Deadline queued or retrying => native pending, with useful retry diagnostics.
+- Active task/startup => native in progress; show measured progress or initialization rather than fabricated percentages.
+- Completion requires a valid committed ComfyUI result. Reconcile Deadline/reporting acknowledgement races; never treat exit code alone as proof of outputs.
+- Exhausted retries, preflight/execution failures, and invalid/missing outputs => native failed, with actionable node or infrastructure errors.
+- User Stop => durable cancellation intent; suspend/stop Deadline scheduling using tested controls, then interrupt/dequeue only the corresponding worker prompt. Confirm actual stop before showing a terminal state. Prevent Deadline from simply retrying the interrupted prompt.
+- Cancelling a terminal run is a no-op. Result publication and cancellation use the same lock so their ordering is deterministic.
+- An externally suspended Deadline job is not automatically “cancelled.” Preserve its diagnosis and define its native representation without inventing a schema state.
+
+The reusable endpoint path currently starts an own-prompt waiter. Killing that waiter alone is not proof that the ComfyUI prompt stopped. Test cancellation and Deadline Worker shutdown hooks explicitly. Never use an unqualified global interrupt on a borrowed endpoint.
+
+Do not persist browser Comfy API auth tokens or credentials in the shared snapshot/metadata/logs. If a workflow needs authenticated services, define worker-side credential provisioning and report missing credentials. Validate job-info fields against newline injection and asset paths against traversal/allowed roots.
+
+## 8. Implementation sequence and exit checks
+
+Do each milestone as a small reviewable change. Record actual results and any changed decision in this document or a linked validation log. Do not call source inspection a live test.
+
+### M0 — Prove the three high-risk seams
+
+- [x] Install a temporary plugin-only native Jobs adapter with a deterministic fake farm provider.
+- [x] Show local work plus two remote runs through pending/running/completed/failed/cancelled, complete workflow detail, outputs, Stop, and retry in the existing UI.
+- [x] Prove per-run progress/previews with concurrent node IDs, active count/filtering/pagination, reconnect, and no local execution/canvas corruption.
+- [x] Prove Run dropdown selection and submission routing while preserving normal widget seeds, keyboard invocation, batch count, and other API wrappers.
+- [x] On a test Deadline Worker, measure cold startup, second-task reuse, cross-job session reuse, cancellation, sandbox teardown, and idle cleanup. Record PID/session identity, model reload/cache behavior for compatible workflows, elapsed time, and GPU memory behavior.
+- [x] From two hosts using the proposed share, prove locking, fencing, atomic publication, crash recovery, and stale-writer rejection.
+
+Exit: demonstrated native integration, viable coordinator-free session ownership, and safe durable state. Do not build the full feature on an unproven seam. A cold-run prototype may proceed for investigation, but warmth remains required for final completion.
+
+### M1 — Durable payloads and one real farm run
+
+- [x] Refactor staging/submission helpers for both the existing node and new Python entry point, preserving node behavior.
+- [x] Implement versioned payload, receipt, control, attempt, and result records; idempotent submission/reconciliation; metadata discovery.
+- [x] Submit one real one-task job, run existing worker preflight, publish validated result metadata, and open its readable workflow/output.
+- [x] Shut down the submitting instance after acceptance and verify the run finishes. Restart and rediscover it from Deadline/shared metadata.
+- [x] Test submit-response loss, crash before activation, crash after activation, and partial batch acceptance.
+
+Exit: an accepted real run is independent of the submitter, with a recoverable native ID and immutable snapshot.
+
+### M2 — Native queue/history, progress, results, and controls
+
+- [x] Replace the fake provider with cached Python Deadline queries and shared result metadata.
+- [x] Complete merged API filtering/pagination/detail/assets and required legacy controls.
+- [x] Connect native Run and finite batches to real submission; add settings and capability diagnostics only, no second job list.
+- [x] Connect native per-job Stop, batch Stop, error display, exact-snapshot Retry, previews, and view/download/Add to workflow.
+- [ ] Test local and remote jobs together, workflow-tab changes, multiple browser tabs, backend restart, Deadline disconnect, and result-cache loss.
+
+Exit: the existing Jobs UI handles the full farm lifecycle without compromising local runs.
+
+### M3 — Persistent worker ownership and idle release
+
+- [x] Implement the session lifecycle selected by M0 with explicit owned/borrowed modes and reservations.
+- [x] Reuse relevant Distributed identity/registration/heartbeat logic without depending on the submitting master.
+- [x] Preserve stable directories, per-attempt staging, GPU affinity, startup options, and environment compatibility checks.
+- [x] Warm reuse across at least three independent jobs; drain on idle; recover from ComfyUI crash and Worker restart; prove no unrelated artist prompt/process is interrupted.
+- [x] Document and test GPU release when other Deadline applications share the Worker.
+
+Exit: warm session reuse and cleanup are supervised and work without the submitting computer.
+
+### M4 — Failure recovery, packaging, and rollout
+
+- [x] Kill a Worker mid-run; allow Deadline retry elsewhere. Publish a late result from the old attempt; only the current accepted result appears.
+- [x] Exercise Stop-versus-completion, Stop-versus-retry, duplicate submission requests, duplicate callbacks, and result-commit/acknowledgement loss.
+- [ ] Exercise missing nodes/models/inputs, browser-interactive nodes, missing output, task timeout, share outage, and incompatible worker session.
+- [x] Test a five-run batch on two GPU Workers with the submitting computer off; no workflow splitting, no missing or duplicate visible results.
+- [x] Deploy backend extension and Deadline helpers together; test from a fresh installed layout. Preserve existing node and Distributed regressions.
+- [x] Document settings, storage/retention, worker setup, supported versions, limitations, transport/credential requirements, and rollback. Remove spike-only fixtures from production loading.
+
+Implementation exit: the live durability, two-Worker batch, recovery, warm ownership, native controls/results, and real progress/preview gates passed. The unchecked validation matrix remains a rollout follow-up; see the linked record for the exact coverage and limits. Disable/rollback native adapters independently if compatibility changes.
+
+## 9. Suggested code ownership
+
+Use the fewest files that keep these responsibilities clear; names below are proposed, not a requirement to build a framework.
+
+- `__init__.py`: register backend integration and `WEB_DIRECTORY`, preserve mappings.
+- `deadline_submit.py`: shared submission/staging helpers and existing node compatibility.
+- `farm_backend.py`: settings, submission, cached Deadline queries, reconciliation, and control endpoints.
+- `native_jobs_adapter.py`: native API response shaping/routing and compatibility checks; no scheduling or worker dispatch.
+- A small shared protocol/store helper: atomic records, locks, generations, metadata validation. Ensure the same implementation is available in the deployed Deadline plugin; test packaging/imports rather than assuming the render sandbox can import this custom-node checkout.
+- `web/deadline_queue.js` and a small versioned compatibility module if needed: Run selection/routing and native progress/preview/retry integration. Keep settings separate from private UI seams.
+- `plugins/ComfyUI/ComfyUI.py` plus a focused session helper if needed: executing assigned jobs, tracking own prompts, reporting/publishing results, and owned process lifetime.
+- Distributed registration/monitor files: only the narrow reuse needed for session identity/liveness. Do not bring browser dispatch or workflow splitting into the new path.
+
+Do not rewrite the existing 2,000+ line Deadline plugin wholesale just to support the new protocol. Preserve its existing tested execution and preflight paths. New behavior should be identified by an explicit protocol/mode marker in job metadata.
+
+## 10. Acceptance and validation
+
+Acceptance checklist (source and automated/live evidence recorded in the linked validation log; existing Distributed has regression coverage but has not been rerun as a full live Collector/Upscale job):
+
+- [x] Only plugin repositories changed; ComfyUI core/frontend files unchanged.
+- [x] One native Jobs/history UI; Deadline-specific UI contains settings/health rather than a duplicate job list.
+- [ ] Normal local Run and existing node/Distributed workflows still work.
+- [x] Resolved seeds/widget values match native submission; explicit Retry preserves the original snapshot.
+- [x] All accepted runs, including waiting runs, complete with the submitter off.
+- [x] Multiple Workers execute independent whole snapshots concurrently through Deadline assignments.
+- [x] Native progress, previews, workflow metadata, outputs, errors, Stop, and retry refer to the correct run.
+- [x] Restart/reconnect restores runs from durable state; local cached history is not the only record.
+- [x] Worker/task retries cannot publish duplicate visible results; Stop cannot cause automatic re-execution.
+- [x] Warm owned sessions persist across jobs and release idle GPU/process resources; borrowed artist sessions remain safe.
+- [x] Unsupported versions/settings and disconnected services produce actionable diagnostics, not silent routing changes.
+
+Meaningful automated tests: merged API filtering/pagination and local pass-through; real response-schema fixtures; submission acknowledgement loss; same-key/different-payload rejection; fencing/cancel/result races; safe result paths; and session ownership/drain rules. Existing stub tests remain useful but cannot replace multi-host SMB and live Deadline tests.
+
+Run the existing suite against the checkout:
+
+```powershell
+& 'C:\AI\ComfyUI_windows_portable4\python_embeded\python.exe' -c "import sys, unittest; sys.path.insert(0, '.'); suite = unittest.defaultTestLoader.discover('tests'); result = unittest.TextTestRunner(verbosity=1).run(suite); sys.exit(not result.wasSuccessful())"
+```
+
+For browser verification, use T3's collaborative preview when available. Record screenshots and concise observed results for M0/M2. Keep any farm test jobs isolated to the chosen test Workers/pool and clean up their owned sessions/files through the implemented lifecycle.
+
+## 11. First actions for the implementation agent
+
+1. Read this plan and applicable repository instructions; inspect current diffs and versions.
+2. Confirm `deadlinecommand`, repository aliases, live ComfyUI, shared root, and test Worker/GPU access. Discover existing configured defaults before requesting missing values.
+3. Start M0 with the native adapter fixture and dropdown/retry/progress prototype. Do not start by building a Farm job panel or a standalone coordinator.
+4. Run the persistent session and two-host storage spikes, record evidence, and settle their ownership/locking decisions before implementing the full durable path.
+5. Continue M1–M4 in order. Keep the accepted product boundaries; report a demonstrated missing seam if a gate fails instead of silently changing the design.
+
+Reference source paths outside this repository are for inspection only. No permission to modify ComfyUI core/frontend is implied by this handoff.

docs/NATIVE_QUEUE_VALIDATION.md

@@ -0,0 +1,121 @@
+# Native queue validation — 2026-10-06
+
+## User test recovery and lifecycle fixes — 2026-10-07
+
+The four MiniMax video tests completed on m21, M23, SIM01 and M25. The submitting host repeatedly read an unlinked, generation-zero SMB record after remote atomic replacement. This hid Worker/progress information, falsely reported successful jobs as failed, and allowed cancellation to overwrite newer attempt metadata. A read with DELETE access alone, and an unbuffered read allowing sharing, both failed repeated two-host validation. Brief exclusive, unbuffered UNC reads passed 30 successive publications from SIM01 to M28; cancellation preserved the current generation/assignment and rejected a late result commit. Evidence is retained under the ignored `work/smb-freshness-config.json` and its isolated NAS validation folder.
+
+All four accepted result pointers and twelve output hashes were verified. The four MP4s also passed size/hash checks when fetched through ComfyUI's native asset URLs. Nine earlier Deadline jobs were confirmed deleted and reconciled to cancelled history without resubmission. The live browser showed zero active jobs, four completed runs and nine cancelled runs. Evidence: `work/lifecycle-recovery-verified.json`.
+
+132 Python tests and 41 JavaScript tests pass. Regressions cover explicit missing-job confirmation versus repository outages, result/attempt fencing, old progress rejection, cancellation event ordering, lost activation followed by deletion or failure, and storage changes after confirmed removal. Windows reads cover BOMs, exact 64 KiB boundaries and larger records. Seven DOM checks cover Stop placement, virtual row reuse, Stopping, settled renders, terminal cleanup and local-row isolation. A fresh isolated cold run was stopped through its native row and reached Cancelled with both task-owned processes exited; evidence: `work/lifecycle-stop-test.json`. The QA run was hidden after verification, preserving its shared records.
+
+The idle artist was restarted through Manager to load these fixes, retaining port 8188 and GPU 1. Only the private NAS02 plugin's shared protocol helper was updated; production repository files and farm Worker settings were unchanged. The additional Opus 5.5 review attempt was rate limited. Read-only code review guided the missing-job fences, cancellation events and Stop DOM fixes.
+
+## Current revision after Opus 5.5 review — 2026-10-07
+
+The sidebar now contains destination (pool/group), priority, and More options for timeout and an optional Worker restriction. Changes save automatically. Machine installation, shared-storage/identity forms, enable toggle and Apply/Revert are removed from the artist interface. The local installer API and farm-mutating setup code are deleted. Normal tasks use existing Worker installation/GPU affinity without a persistent owner. Warm owners remain optional administration.
+
+124 Python tests and 40 JavaScript tests passed after review fixes. New tests cover cold-owner affinity, PID rejection, task-owner cleanup after broker failure, Worker backend isolation, task-derived startup budgets, immediate missing-plugin errors, unrestricted jobs, real Deadline ownership independent of history scope, existing Workers without mappings, atomic execution-mode claims, cancellation winning over late startup failures, and avoiding control commands for missing or already stopped jobs.
+
+Live sidebar run `33c63a5c-13cf-4631-9fe3-5f47a2fe972b` / Deadline job `6ac576cd961826ce29de4037` completed on the isolated local test Worker with its persistent owner stopped/disabled and its session mapping absent. A task-owned supervisor launched ComfyUI on GPU 0; startup took 45.125 seconds. The accepted PNG is 64 × 64, RGB (51,119,187), hash verified and available through native Jobs assets. Supervisor PID 54536 and GPU child PID 25072 both exited after completion. Deadline recorded the real user `dominik.bargiel`. Evidence: ignored `work/unslop-cold-render.json`.
+
+The missing-node cold test `e8a7ef0b-c3f7-45ca-bc6b-c6c0d0863fb7` / Deadline job `6ac578db120b5077e2f06dc2` reached Deadline Failed after three assignments. Each attempt released its owned processes; no result was committed. Evidence: `work/unslop-cold-failure-cancel.json`.
+
+Fresh startup cancellation `8ec52c3a-e75e-4434-96d4-e5c5bffb8fd4` / job `6ac57b1a0f12eb2d77330bd2` automatically reached Cancelled in durable control and native Jobs, committed no result, and released supervisor PID 34204 and child PID 23076. The merged queue was empty afterward. Evidence: `work/unslop-cold-cancel-fixed.json`. Earlier cancellation exposed missing terminal reconciliation and a late startup transport error overwriting cancellation. Those were fixed and covered by regression tests; the original stuck test record was repaired only after proving its Deadline task stopped and processes exited, and is explicitly recorded as an operator repair rather than live proof of the fix.
+
+Live browser checks confirmed autosave (50 → 51 → 50), Run gating while saving, selection retained during search, no artist setup forms, visible custom-plugin override, and a 312 px sidebar with a 57.5 px footer. T3 captured an earlier screenshot, but final snapshot calls failed; the final inline visual uses the actual live sidebar DOM and production stylesheet and is labelled as a DOM capture. Testing used a separate blank workflow; the original ten-node graph and tracked workflow state were retained and the test workflow was closed.
+
+Only the existing isolated test Worker and private NAS02 plugin were changed for this verification. The private plugin pins the local portable4 installation for isolation. Production source still resolves Deadline's ComfyUIPath; the repository plugin and original farm Workers were untouched. Production rollout requires deploying the updated helpers and configuring NativeQueueRoot. Reservation is not implemented; optional warmth does not reserve Workers.
+
+## Previous sidebar and local setup (superseded)
+
+The sidebar now separates **Workers** and **Setup**, filters candidates by both pool and group, keeps invalid selections visible, and shows the full farm only when explicitly requested. The verified 312 px panel has one scrolling region and a 55.5 px normal footer. Draft settings block Run until Apply or Revert; backend readiness gates submission independently of the UI.
+
+Final review also added durable-record checks before changing storage or updating setup, including lost activation responses and a restarted backend with no cache. New captures and background receipt recovery cannot race that check. Distributed collector/upscaler workflows are rejected before their browser dispatcher; explicit local Distributed Run keeps its existing behavior.
+
+The final suite passes **138 Python tests and 48 JavaScript tests**. It includes restricted Worker user-scope eligibility, unknown scheduling modes, correction of never-initialized roots, inaccessible-share preservation, and settings/receipt concurrency regressions. The intentional existing synthetic queue-failure traceback remains expected.
+
+The actual Setup action on port 8188 created **M28-ComfyUI-GPU0** with an owned GPU-0 session, the `test` pool, `comfy` group, and the isolated `dominik.bargiel-comfy` scope. Storage is `\\NAS02\Comfy\playground\deadline-native-queue`; its private plugin lives under `plugin\ComfyUI`. The repository's production ComfyUI plugin and existing farm Workers were not changed. The two limited interactive scheduled tasks remain installed as the usable local destination.
+
+Clicking the sidebar's **Run on Deadline** queued run `1165f5b7-540c-43ad-89b9-e3740c52b15c` through the native Run controls. Its EmptyImage/SaveImage workflow completed on the dedicated Worker. The accepted PNG is 64 x 64, has the expected RGB pixel `(136, 85, 204)`, and matches its result-manifest hash. The native Jobs assets endpoint serves the result. After completion the owner had no child PID and no reservation. Evidence is `work/final-sidebar-render.json`.
+
+Sixteen additional real-browser checks covered relevant Worker filtering, compact footer, hidden clean draft actions, draft gating, refresh/Revert, retained out-of-group selections, explicit entire-farm discovery, actual storage verification, and final reload/restoration. Evidence is `work/final-sidebar-browser-checks.json`. The browser's original ten-node workflow, positions, connections, and widget values were verified restored; the review-created duplicate tab was closed. After the final backend restart the completed run and its assets were recovered, readiness remained true, and a real API request using a disallowed Worker scope returned HTTP 400 without changing saved settings. T3 evaluate worked; snapshot calls repeatedly returned `PreviewAutomationExecutionError`, so a new live browser screenshot is not claimed. The inline preview is a read-only rendering of the verified sidebar DOM and its production stylesheet.
+
+The sections below retain earlier implementation evidence and validation history; their historical process IDs, counts, and UI descriptions do not describe the current sidebar.
+
+## Earlier implementation evidence
+
+Implementation is present in this checkout, uncommitted. Native mode remains opt-in. All execution tests ran on **M28**, using a restricted test user and the `test` pool. A temporary second Deadline Worker on M28 exercised GPU 1 through the borrowed artist endpoint. **SIM01 was used only for the two-host SMB probe**, with no ComfyUI plugin synchronization or GPU-job interruption there.
+
+Baseline: plugin `e87701a` / 1.0.10; Distributed `62c81b1`; ComfyUI 0.39.0; frontend 1.53.10; Deadline 10.4.2.2. The artist on port 8188, PID 17284, remained alive throughout the final tests. Source changes are confined to this plugin. Core status also shows a deleted runtime output placeholder; no core/frontend/Distributed source was edited.
+
+## Automated evidence
+
+**78 Python tests and 9 JavaScript tests pass.** This includes the original 30 regressions. Coverage includes merged native API filtering/pagination, local pass-through, bounded finite batches, resolved seeds, interleaved request routing, captured API wrappers, recursive/partial submission rejection, response/activation loss, idempotency, scope/settings checks, exact retry, output-cache rebuild, cancellation/result races, duplicate callbacks, stale assignments down to one .NET tick, safe asset paths, missing outputs, task timeout, disabled-mode retry rejection, actual execution-queue isolation, owned/borrowed drain rules, and Deadline-unavailable ownership preservation.
+
+The intentional synthetic queue-failure traceback in the existing Python suite is expected. These tests use stubs where farm or browser failures must be controlled; their success is not presented as a live network failure test.
+
+```powershell
+& 'C:\AI\ComfyUI_windows_portable4\python_embeded\python.exe' -s -c "import sys,unittest;sys.path.insert(0,'.');r=unittest.TextTestRunner().run(unittest.defaultTestLoader.discover('tests'));sys.exit(not r.wasSuccessful())"
+node --test tests/*.mjs
+```
+
+## Deadline sidebar revision
+
+After the native queue gates, the configuration UI was moved to a dedicated public-API sidebar, following Distributed's registration pattern. The revised suite passes **83 Python tests and 14 JavaScript tests**, including discovery caching, stale-data retention, settings draft preservation, failed saves, aborted refreshes, and optimistic settings conflicts.
+
+The isolated CPU ComfyUI on port 8197 passed **23 browser checks** against the actual frontend and repository. These cover registration, 80 discovered Workers, search, offline selection retention, Apply/Revert, draft preservation across refresh and reopening, validation failure, external edits detected during refresh, a concurrent-save HTTP 409, native Jobs opening, setup diagnostics, and layout at 240–420 px widths. The footer remained within both 800 px and 650 px viewports. Browser-generated evidence is retained at `work/sidebar-qa-final.json`; the opt-in harness is `work/sidebar_qa.js` and is outside production extension loading.
+
+The Run bridge check temporarily supplied configured discovery metadata and intercepted the native Run click before execution; it proves delegation to native controls, not a new live GPU run or session-owner health check. All new farm discovery calls are read-only. T3 navigation and resizing worked; snapshot/evaluate calls failed, so no new sidebar screenshot is claimed. The main artist process (now PID 54004) remained running; only isolated CPU servers were restarted. Restart the artist normally and refresh its browser to load the new discovery/settings routes and sidebar. The isolated settings were restored and the test server stopped after verification.
+
+## Live results
+
+The follow-up live sidebar check restarted the idle artist through ComfyUI Manager to resolve a running-backend/browser-code mismatch. The discovery route on port 8188 then returned `connected=true`, repository `\\NAS01\repository`, 80 Workers, and five pools. The evidence is `work/sidebar-live-8188-discovery.json`. The artist kept port 8188 and GPU 1; Manager replaced PID 54004 with PID 55284. Native submission remained disabled and no jobs were submitted. Four additional JavaScript regression tests bring the total to **18**, covering the missing-route restart diagnostic and repository connectivity independent of submission enablement.
+
+T3 browser inspection of the live sidebar confirmed **Connected**, `0 selected · 80 in farm`, and the actual repository pool choices. A saved capture is `docs/images/deadline-sidebar-connected.png`. The native plugin deployment warning remains visible separately from the successful repository connection.
+
+Evidence is retained under `\\NAS02\Comfy\playground\deadline-native-queue-tests`:
+
+| Gate | Observation | Evidence beneath that root |
+| --- | --- | --- |
+| Two-host SMB | M28/SIM01 mutually excluded writers; 2,370 unlocked atomic reads; remote process death released its lock; stale publication rejected | `storage-20261006-5/validation.json` |
+| Three independent jobs, warm owned session | One supervised PID 48564 / session `bb574cfd-c022-4c66-82ad-db6e1424470c` across three Deadline sandboxes; task times 10.688, 0.188, 0.219 seconds; model loaded once | `sessions-20261006-3` and `direct-sessions-20261006` |
+| Targeted interrupt and owner death | Only the reserved prompt was interrupted; owner death removed owned child 49448; replacement owner could start another child; artist identity preserved | `session-recovery-20261006/validation.json` |
+| Durable acceptance and restart | Three accepted jobs completed after isolated submitting ComfyUI PID 30268 was killed; restarted submitter recovered results and served `/view` | `durable-20261006-4/validation.json` |
+| Five native Run batch items on two GPU Workers | All five were accepted before submitter PID 33004 was killed; GPU 0 owned session rendered two, GPU 1 borrowed endpoint rendered three; five distinct valid results | `durable-20261006-4/two-workers-validation.json` |
+| Stop and exact retry | Pending Stop never rendered; running Stop targeted its own Worker prompt and committed no result; retry retained the source prompt and duplicate request key returned the same new run | `durable-20261006-4/controls-validation.json` |
+| Shared Worker GPU handoff | Owned session had no PID/reservation before task completion; borrowed artist survived | `durable-20261006-4/controls-validation.json` |
+| Worker killed mid-render | Second GPU Worker completed generation 2 with eight files; a late generation-1 publication returned false and left the accepted pointer unchanged | `durable-20261006-4/worker-crash-validation.json` |
+| Native output reuse | Add to current workflow created an annotated LoadImage; normal local Run completed; staged farm LoadImage/SaveImage round trip preserved image pixels. Fresh copied helper verified exact task-start ticks before commit | `durable-20261006-4/final-roundtrip-validation.json` |
+| Native workflow and Retry | Open as workflow restored the readable graph; native context-menu Retry created `b5027c5c-1823-4b0e-82d6-00140168ed00`, which completed through the fresh helper | Same final-roundtrip report; source/new `submission.json` |
+| Real sampler progress and preview | SDXL base, seed 777, 40 Euler steps at 384×384 completed. Observed 32 node-state samples and live JPEG previews with changing hashes; one accepted PNG | `durable-20261006-4/real-preview-validation.json` |
+| Storage authority unavailable | Temporarily removing only the owned test store's sentinel retained all 17 cached jobs; submit returned 503 with a storage diagnostic; restoration reconnected | `durable-20261006-4/storage-outage-validation.json` |
+
+Worker crash recovery used explicit Deadline requeue after the kill, rather than waiting for the repository's Worker-loss detection timeout. Both assignments were real Deadline assignments. The storage-outage gate injected a missing store identity; it did not disconnect the NAS or disrupt other users.
+
+The M0 fixture was also exercised through the installed frontend: local plus remote statuses, readable workflow/output, Stop/retry, keyboard/dropdown focus, finite batch seeds 100/101/102, exact stored retry while widgets had advanced, two concurrent runs using the same node ID with different progress/previews, 209 pending rows across pagination, and reconnect. Fixture providers are loaded only by their explicit spike server.
+
+An early Distributed wrapper bypass was found during M0; own accidental local prompts were targeted and removed/finished. Routing now marks the serialized request before captured API wrappers, and the Python `/prompt` guard rejects farm markers. No accepted production farm run enters the submitting instance's execution queue.
+
+One T3 screenshot was saved and inspected. It shows the existing native Jobs panel with farm progress, completed outputs, and correctly labelled cancelled records:
+
+![Native Jobs validation](images/native-queue-validation.png)
+
+Further screenshot calls intermittently failed with `PreviewAutomationExecutionError`; browser DOM/API observations and the saved screenshot are the evidence available. Screenshot failure is not recorded as a UI failure. After the real sampler finished, native `activeJobId` was null and global/per-job progress and preview state were empty.
+
+## Corrections established by live tests
+
+- Independent scheduled Worker/owner processes avoid the tool process's Windows process-tree restrictions.
+- Portable Python uses `-s` to avoid an older user-site Torch stack shadowing the installation.
+- Deadline's `CustomPluginDirectory` names the parent containing the plugin-name directory; copied helper imports explicitly add their sandbox directory.
+- Windows temp-directory arguments need ComfyUI's appended `temp` subfolder accounted for.
+- Native retained inputs use shallow deterministic content names; nested result references otherwise exceeded Deadline Python's Windows path limit.
+- Owned sessions enable automatic sampler previews by default. Finished reservations remain held while Deadline is still publishing their results.
+- Missing shared authority keeps cached history. Terminal progress has a newer sequence than its final live sample, so stale running previews clear.
+
+## Validation limits and cleanup
+
+Core/front-end compatibility is intentionally limited to the tested versions. Independent multi-browser configuration changes, every custom loader/output type, credential-dependent workflows, large-scale farm load, physical SMB cable loss, and the farm's automatic Worker-loss timing were not exhaustively exercised. Missing nodes/browser-interactive behavior, timeout, and missing output have automated negative coverage; they were not all repeated as live GPU failures. SDXL is a representative real sampler test, not an environment certification for every model.
+
+The final cleanup record is `test-job-cleanup.json` at the evidence root. Completed/failed/suspended test jobs and off-host evidence are retained for review; no test job is left active. The temporary Worker is deleted, scheduled owners/probes are unregistered, and isolated test servers are stopped. The two test plugin directories were retired as `.disabled-native-20261006` backups beside the production plugin. Automatic approval review rejected their deletion and deletion of three inactive SIM01 Public probe files with a policy block; those small files are retained and have no scheduled task. M28's scheduling/user/concurrency/registration settings are restored. The production repository ComfyUI plugin was not overwritten, and the artist process was not restarted.
+
+Use [the setup guide](DEADLINE_NATIVE_QUEUE.md) to deploy and enable native mode deliberately on the existing installation. No commit was created.

farm_backend.py

@@ -0,0 +1,845 @@
+"""Durable Deadline submission and a cached provider for ComfyUI's native Jobs.
+
+All filesystem/Deadline work runs outside aiohttp's event loop. The share is
+authoritative; this provider's cache can be discarded and rebuilt on restart.
+"""
+import asyncio
+import copy
+import getpass
+import json
+import mimetypes
+import os
+import re
+import threading
+import time
+import uuid
+import shutil
+from urllib.parse import urlencode
+from pathlib import Path
+
+from aiohttp import web
+
+try:
+    from .deadline_submit import DeadlineCommandHelper, DeadlineSubmitNode, InputAssetStager
+    from .farm_protocol import RunStore, SCHEMA, TERMINAL, atomic_json, canonical_id, digest, read_json, record_lock, safe_path
+    from .native_jobs_adapter import NativeJobsAdapter
+    from .farm_readiness import off_host_root, readiness as evaluate_readiness
+except ImportError:
+    from deadline_submit import DeadlineCommandHelper, DeadlineSubmitNode, InputAssetStager
+    from farm_protocol import RunStore, SCHEMA, TERMINAL, atomic_json, canonical_id, digest, read_json, record_lock, safe_path
+    from native_jobs_adapter import NativeJobsAdapter
+    from farm_readiness import off_host_root, readiness as evaluate_readiness
+
+DEFAULTS = {"enabled": False, "root": "", "scope": getpass.getuser(), "pool": "none", "group": "none",
+    "priority": 50, "workers": "", "machine_limit": 1, "task_timeout_minutes": 30, "plugin_directory": ""}
+SETTINGS_KEYS = set(DEFAULTS)
+DISTRIBUTED_NODE_TYPES = {"DistributedCollector", "DeadlineDistributedCollector", "DistributedVideoCollector",
+    "DeadlineDistributedVideoCollector", "UltimateSDUpscaleDistributed", "DeadlineUltimateSDUpscaleDistributed"}
+
+
+class SettingsConflict(ValueError):
+    pass
+
+
+def field(value):
+    value = str(value)
+    if any(c in value for c in "\r\n\0"):
+        raise ValueError("Deadline fields cannot contain newlines or NUL")
+    return value
+
+
+def validate_settings(value):
+    if not isinstance(value, dict) or set(value) - SETTINGS_KEYS:
+        raise ValueError("Unknown Deadline setting")
+    settings = {**DEFAULTS, **value}
+    if type(settings["enabled"]) is not bool:
+        raise ValueError("enabled must be a boolean")
+    for key in ("scope", "pool", "group", "workers", "root", "plugin_directory"):
+        settings[key] = field(settings[key]).strip()
+    if not settings["scope"] or len(settings["scope"]) > 120:
+        raise ValueError("A user scope of 1 to 120 characters is required")
+    for key, low, high in (("priority", 0, 100), ("machine_limit", 1, 10000), ("task_timeout_minutes", 1, 1440)):
+        if type(settings[key]) is not int or not low <= settings[key] <= high:
+            raise ValueError(f"{key} must be an integer between {low} and {high}")
+    if settings["enabled"] and not off_host_root(settings["root"]):
+        raise ValueError("Native queue requires an off-host UNC root")
+    if settings["enabled"] and settings["root"].split("\\")[2].lower() in {"localhost", "127.0.0.1", os.environ.get("COMPUTERNAME", "").lower()}:
+        raise ValueError("Native queue storage must be off the submitting host")
+    if settings["plugin_directory"] and not off_host_root(settings["plugin_directory"]):
+        raise ValueError("Custom plugin directory must be an off-host UNC folder containing ComfyUI")
+    return settings
+
+
+def parse_ini(output):
+    records, current = {}, None
+    for line in output.splitlines():
+        if line.startswith("[") and line.endswith("]"):
+            current = records.setdefault(line[1:-1], {})
+        elif current is not None and "=" in line:
+            key, value = line.split("=", 1)
+            current[key] = value
+    return records
+
+
+def deadline(*args):
+    result = DeadlineCommandHelper.call_deadline_command(list(args), timeout=30)
+    if any(line.startswith("Error:") for line in result.splitlines()):
+        raise RuntimeError(result.strip())
+    return result
+
+
+class FarmBackend:
+    def __init__(self, settings_path=None, command=deadline):
+        self.settings_path = Path(settings_path or os.environ.get("COMFY_DEADLINE_NATIVE_SETTINGS") or Path(__file__).parent / "deadline_native_settings.json")
+        self.command = command
+        self.plugin_directory = field(os.environ.get("COMFY_DEADLINE_NATIVE_PLUGIN_DIR", ""))
+        self.lock = threading.RLock()
+        self.configuration_lock = threading.RLock()
+        self.configuration_in_progress = False
+        self.active_submissions = 0
+        configuration_error = None
+        try:
+            self.settings = validate_settings(read_json(self.settings_path)) if self.settings_path.exists() else dict(DEFAULTS)
+        except (OSError, ValueError, TypeError) as exc:
+            self.settings = dict(DEFAULTS)
+            configuration_error = "Native settings could not be loaded: " + str(exc)
+        self.jobs, self.histories, self.assets, self.progress = {}, {}, {}, []
+        self.health = {"connected": False, "message": configuration_error or "Native queue is disabled; configure an off-host root and Workers.", "last_refresh": None}
+        self.wakeup = threading.Event()
+        self.shutdown = threading.Event()
+        self.thread = None
+        self.result_cache = {}
+        self.discovery_lock = threading.Lock()
+        self.discovery_checked = 0
+        self.discovery_plugin_directory = None
+        self.discovery = {"connected": False, "message": "Not checked yet", "workers": [], "pools": [], "groups": [], "last_refresh": None}
+        self.readiness_cache = evaluate_readiness(self.settings, self.discovery)
+        try:
+            import folder_paths
+            self.output_root = Path(folder_paths.get_output_directory())
+        except (ImportError, AttributeError):
+            self.output_root = None
+
+    def _effective_plugin_directory(self, settings):
+        return settings["plugin_directory"] or self.plugin_directory
+
+    def discover(self, force=False, plugin_directory=None):
+        if plugin_directory is None:
+            with self.lock:
+                plugin_directory = self._effective_plugin_directory(self.settings)
+        requested = time.monotonic()
+        with self.discovery_lock:
+            same_plugin = self.discovery_plugin_directory == plugin_directory
+            if same_plugin and self.discovery_checked and (requested - self.discovery_checked < 15 and not force or self.discovery_checked > requested):
+                return copy.deepcopy(self.discovery)
+            try:
+                arguments = ["-ExecuteScript", str(Path(__file__).parent / "scripts" / "deadline_farm_snapshot.py")]
+                if plugin_directory:
+                    arguments.append(plugin_directory)
+                output = self.command(*arguments)
+                lines = [line[len("COMFY_NATIVE_FARM="):] for line in output.splitlines() if line.startswith("COMFY_NATIVE_FARM=")]
+                if len(lines) != 1:
+                    raise ValueError("Deadline did not return a farm snapshot")
+                snapshot = json.loads(lines[0])
+                if not isinstance(snapshot, dict) or any(not isinstance(snapshot.get(key), list) for key in ("workers", "pools", "groups")):
+                    raise ValueError("Invalid Deadline farm snapshot")
+                snapshot["deadline_user"] = field(self.command("-GetCurrentUserName").strip())
+                self.discovery = {**snapshot, "connected": True, "message": "Repository connected", "last_refresh": int(time.time() * 1000)}
+            except Exception as exc:
+                self.discovery.update(connected=False, message=str(exc))
+                if not same_plugin:
+                    self.discovery.update(plugin_available=False, plugin_directory=plugin_directory)
+            self.discovery_checked = time.monotonic()
+            self.discovery_plugin_directory = plugin_directory
+            return copy.deepcopy(self.discovery)
+
+    def readiness(self, settings=None, force=False):
+        if settings is None:
+            with self.lock:
+                settings = dict(self.settings)
+        else:
+            settings = validate_settings(settings)
+        snapshot = self.discover(force, self._effective_plugin_directory(settings))
+        storage = {"connected": False, "message": "Choose an off-host shared folder."}
+        if off_host_root(settings["root"]):
+            try:
+                RunStore(settings["root"]).check_root()
+                storage = {"connected": True, "message": "Shared storage identity is available."}
+            except (OSError, ValueError, RuntimeError) as exc:
+                storage["message"] = str(exc)
+        result = evaluate_readiness(settings, {**snapshot, "storage": storage})
+        with self.lock:
+            if settings == self.settings:
+                self.readiness_cache = result
+        return result
+
+    def _require_ready(self, settings):
+        result = self.readiness(settings)
+        if not result["ready"]:
+            if result["code"] in {"storage", "repository"}:
+                raise RuntimeError(result["message"])
+            raise ValueError(result["message"])
+        return result
+
+    def configure(self, value, expected=None):
+        with self.configuration_lock:
+            return self._configure(value, expected)
+
+    def _configure(self, value, expected=None):
+        settings = validate_settings(value)
+        with self.lock:
+            if expected is not None and expected != self.settings:
+                raise SettingsConflict("Settings changed elsewhere. Revert to reload before applying.")
+            self._check_storage_change(settings)
+            previous = dict(self.settings)
+            changing_store = (settings["root"], settings["scope"]) != (previous["root"], previous["scope"])
+            if changing_store:
+                self.configuration_in_progress = True
+        try:
+            if changing_store:
+                self._check_durable_runs(previous)
+            return self._write_configuration(settings, expected)
+        finally:
+            if changing_store:
+                with self.lock:
+                    self.configuration_in_progress = False
+
+    def _write_configuration(self, settings, expected):
+        initializing = settings["enabled"] and (not self.settings["enabled"] or
+            (settings["root"], settings["scope"]) != (self.settings["root"], self.settings["scope"]))
+        if initializing:
+            root = Path(settings["root"])
+            root.mkdir(parents=True, exist_ok=True)
+            with record_lock(root / "health-probe"):
+                atomic_json(root / "health-probe" / "probe.json", {"schema": SCHEMA, "time": time.time()})
+            self.command("-GetRepositoryPath")
+            RunStore(root).initialize()
+            ready = self._require_ready(settings)
+        else:
+            ready = evaluate_readiness(settings, {**self.discovery, "storage":self.health})
+        with self.lock:
+            if expected is not None and expected != self.settings:
+                raise SettingsConflict("Settings changed elsewhere. Revert to reload before applying.")
+            self._check_storage_change(settings)
+            if (settings["root"], settings["scope"]) != (self.settings["root"], self.settings["scope"]):
+                self.jobs, self.histories, self.assets, self.progress = {}, {}, {}, []
+                self.result_cache.clear()
+            atomic_json(self.settings_path, settings)
+            self.settings = settings
+            self.readiness_cache = ready if settings["enabled"] else evaluate_readiness(settings, self.discovery)
+        self.wakeup.set()
+        return settings
+
+    def _check_storage_change(self, settings):
+        if (settings["root"], settings["scope"]) != (self.settings["root"], self.settings["scope"]):
+            if self.active_submissions or any(j["status"] not in TERMINAL for j in self.jobs.values()):
+                raise ValueError("Wait for active submissions and runs before changing the shared root or user scope")
+
+    def _check_durable_runs(self, settings):
+        if not settings["root"]:
+            return
+        if not settings["enabled"] and not off_host_root(settings["root"]):
+            # Disabled local/malformed roots cannot contain accepted native runs.
+            return
+        store = RunStore(settings["root"])
+        try:
+            store.check_root()
+        except RuntimeError:
+            # Verify the share is reachable before treating a new folder as empty.
+            if not settings["enabled"]:
+                directory = store.root
+                share = Path(directory.anchor)
+                while True:
+                    try:
+                        entries = {path.name for path in directory.iterdir()}
+                    except FileNotFoundError:
+                        if directory == share or directory.parent == directory:
+                            raise RuntimeError("The configured shared folder is unavailable; reconnect the share before changing storage or updating setup") from None
+                        directory = directory.parent
+                        continue
+                    if directory != store.root or "store.json" not in entries and "runs" not in entries:
+                        return
+                    break
+            raise
+        pending = []
+        for submission in store.records(settings["scope"]):
+            run_id = submission["run_id"]
+            control = store.control(run_id)
+            if control.get("result") or control["status"] == "cancelled" or control.get("deadline_removed"):
+                continue
+            receipt_path = store.run(run_id) / "receipt.json"
+            receipt = read_json(receipt_path) if receipt_path.exists() else {}
+            if not receipt.get("job_id"):
+                raise ValueError("Resolve pending or unconfirmed farm runs before changing storage or updating setup: " + run_id)
+            pending.append((run_id, receipt["job_id"]))
+        remote = {}
+        for offset in range(0, len(pending), 100):
+            remote.update(parse_ini(self.command("-GetJob", ",".join(job_id for _, job_id in pending[offset:offset + 100]), "True")))
+        for run_id, job_id in pending:
+            if remote.get(job_id, {}).get("Status") not in {"Completed", "Failed"}:
+                raise ValueError("Finish pending or unconfirmed farm runs before changing storage or updating setup: " + run_id)
+
+    def start(self):
+        self.thread = threading.Thread(target=self._refresh_loop, name="Deadline-native-reconciler", daemon=True)
+        self.thread.start()
+
+    def _refresh_loop(self):
+        while not self.shutdown.is_set():
+            try:
+                if not self.settings_path.exists():
+                    site = self.discover()
+                    root = os.environ.get("COMFY_DEADLINE_NATIVE_ROOT", "") or site.get("native_root", "")
+                    if root and site.get("deadline_user"):
+                        self.configure({**self.settings, "root":root, "scope":site["deadline_user"], "enabled":True})
+                self.refresh()
+                with self.lock:
+                    enabled = self.settings["enabled"]
+                if enabled:
+                    self.readiness()
+            except Exception as exc:
+                with self.lock:
+                    self.health.update(connected=False, message=str(exc))
+                    snapshot = {**self.discovery, "storage": self.health}
+                    self.readiness_cache = evaluate_readiness(self.settings, snapshot)
+            self.wakeup.wait(3)
+            self.wakeup.clear()
+
+    def submit(self, body, captured=None, retry_of=None):
+        if not isinstance(body, dict) or not isinstance(body.get("prompt"), dict) or not body["prompt"]:
+            raise ValueError("A non-empty API prompt is required")
+        with self.lock:
+            if self.configuration_in_progress:
+                raise ValueError("Shared storage configuration is changing. Wait before submitting.")
+            settings = validate_settings(captured or {**self.settings, **body.get("settings", {})})
+            if not self.settings["enabled"] or not settings["enabled"]:
+                raise ValueError("Native Deadline queue is disabled")
+            # Retried snapshots and browser captures belong to the current store.
+            if any(settings[k] != self.settings[k] for k in ("root", "scope", "enabled")) or (captured is None and settings["plugin_directory"] != self.settings["plugin_directory"]):
+                raise ValueError("Change storage settings through the settings endpoint")
+            self.active_submissions += 1
+        try:
+            return self._submit(body, settings, retry_of)
+        finally:
+            with self.lock:
+                self.active_submissions -= 1
+
+    def _submit(self, body, settings, retry_of):
+        key = canonical_id(body.get("submission_key"))
+        batch = canonical_id(body["batch_id"]) if body.get("batch_id") else str(uuid.uuid4())
+        if body.get("partial_execution_targets"):
+            raise ValueError("Run (Deadline) supports whole workflows only")
+        if any(not isinstance(n, dict) or not isinstance(n.get("inputs"), dict) or not n.get("class_type") for n in body["prompt"].values()):
+            raise ValueError("Malformed API prompt node")
+        if any(n["class_type"] in {"DeadlineSubmit", "SaveAndSubmitNode"} for n in body["prompt"].values()):
+            raise ValueError("Use local Run for workflows containing Submit to Deadline")
+        if any(n["class_type"] in DISTRIBUTED_NODE_TYPES for n in body["prompt"].values()):
+            raise ValueError("Use local Run for workflows containing Distributed nodes; Deadline queues whole workflows")
+        extra = copy.deepcopy(body.get("extra_data", {}))
+        extra.pop("auth_token_comfy_org", None)
+        extra.pop("api_key_comfy_org", None)
+        workflow = extra.get("extra_pnginfo", {}).get("workflow")
+        if not isinstance(workflow, dict) or "__deadline_native_queue" in workflow:
+            raise ValueError("A readable workflow without routing markers is required")
+        payload = {"prompt": copy.deepcopy(body["prompt"]), "extra_data": extra}
+        fingerprint = digest({"payload": payload, "settings": settings, "batch_id": body.get("batch_id"), "retry_of": retry_of})
+        self._require_ready(settings)
+        store = RunStore(settings["root"])
+        store.check_root()
+        key_dir = store.key(settings["scope"], key)
+        with record_lock(key_dir, timeout=35):
+            request_path = key_dir / "request.json"
+            if request_path.exists():
+                request = read_json(request_path)
+                if request["hash"] != fingerprint:
+                    raise ValueError("Same submission_key with a different snapshot or settings")
+                run_id = request["run_id"]
+            else:
+                run_id = str(uuid.uuid4())
+                atomic_json(request_path, {"run_id": run_id, "hash": fingerprint})
+            directory = store.run(run_id)
+            if not (directory / "submission.json").exists():
+                directory.mkdir(parents=True, exist_ok=True)
+                source_input = str(store.run(retry_of) / "inputs") if retry_of else None
+                stager = InputAssetStager(str(directory / "outputs"), "Native queue", run_id, str(directory / "inputs"), source_input, native_assets=True)
+                _input, manifest, assets = stager.stage_referenced_assets(payload["prompt"])
+                rewriter = DeadlineSubmitNode()
+                rewriter._rewrite_prompt_asset_references(payload["prompt"], assets)
+                rewriter._rewrite_standard_workflow_assets(workflow, assets)
+                for asset in assets:
+                    asset["sha256"] = self._hash_file(Path(asset["destination"]))
+                atomic_json(directory / "input_manifest.json", {"schema": SCHEMA, "assets": assets})
+                if Path(manifest) != directory / "input_manifest.json":
+                    Path(manifest).unlink()
+                created = int(time.time() * 1000)
+                payload["extra_data"]["create_time"] = created
+                submission = {"schema": SCHEMA, "run_id": run_id, "submission_key": key, "batch_id": batch,
+                    "scope": settings["scope"], "deadline_user": field(self.command("-GetCurrentUserName").strip()),
+                    "created_at": created, "workflow_id": workflow.get("id"),
+                    "settings": settings, "payload": payload, "snapshot_hash": digest(payload),
+                    "request_hash": fingerprint, "retry_of": retry_of, "name": "Comfy Native " + run_id}
+                submission["plugin_directory"] = self._effective_plugin_directory(settings)
+                atomic_json(directory / "prompt.json", payload["prompt"])
+                atomic_json(directory / "workflow.json", workflow)
+                atomic_json(directory / "control.json", {"schema": SCHEMA, "generation": 0, "status": "pending",
+                    "cancel_requested": False, "result": None})
+                atomic_json(directory / "submission.json", submission)
+            submission = read_json(directory / "submission.json")
+            with record_lock(directory):
+                receipt = self._ensure_submitted(store, submission)
+            if not receipt.get("accepted"):
+                raise RuntimeError("Submission is unconfirmed; retry this request key to reconcile without duplicating it")
+            self._cache_submission(submission, receipt)
+        self.wakeup.set()
+        return {"prompt_id": run_id, "number": 0, "node_errors": {}}
+
+    def _cache_submission(self, submission, receipt):
+        run_id = submission["run_id"]
+        with self.lock:
+            if run_id in self.jobs:
+                return
+            self.jobs[run_id] = {"id": run_id, "status": "pending", "create_time": submission["created_at"],
+                "workflow_id": submission["workflow_id"], "outputs_count": 0, "previewable_outputs_count": 0,
+                "workflow": submission["payload"], "outputs": {}, "deadline": {"job_id": receipt["job_id"], "accepted": True}}
+            self.assets[run_id] = []
+            self.histories[run_id] = {"prompt": [0, run_id, submission["payload"]["prompt"], submission["payload"]["extra_data"], list(submission["payload"]["prompt"])],
+                "outputs": {}, "status": {"completed": False, "status_str": "success", "messages": []}}
+
+    @staticmethod
+    def _hash_file(path):
+        import hashlib
+        value = hashlib.sha256()
+        with path.open("rb") as handle:
+            for block in iter(lambda: handle.read(1024 * 1024), b""):
+                value.update(block)
+        return value.hexdigest()
+
+    def _ensure_submitted(self, store, submission):
+        directory = store.run(submission["run_id"])
+        receipt_path = directory / "receipt.json"
+        receipt = read_json(receipt_path) if receipt_path.exists() else {"schema": SCHEMA, "accepted": False, "phase": "prepared"}
+        if receipt.get("phase") == "removed":
+            return receipt
+        if receipt.get("accepted"):
+            return receipt
+        if not receipt.get("job_id") and receipt["phase"] == "submitting":
+            matches = parse_ini(self.command("-GetJobsFilterIniAnd", "Name=" + submission["name"], "Plugin=ComfyUI"))
+            matches = {i: j for i, j in matches.items() if self.command("-GetJobExtraInfoKeyValue", i, "ComfyNativeRunId").strip() == submission["run_id"]}
+            if len(matches) > 1:
+                raise RuntimeError("Duplicate Deadline mappings require operator reconciliation")
+            if not matches:
+                return receipt  # Never blindly resubmit an ambiguous operation.
+            receipt["job_id"] = next(iter(matches))
+            atomic_json(receipt_path, receipt)
+        if not receipt.get("job_id"):
+            settings = submission["settings"]
+            job = {"Plugin": "ComfyUI", "Name": submission["name"], "BatchName": "ComfyUI " + submission["batch_id"],
+                "UserName": submission.get("deadline_user", settings["scope"]), "Pool": settings["pool"], "Group": settings["group"],
+                "Priority": settings["priority"],
+                "InitialStatus": "Suspended", "Frames": 0, "ChunkSize": 1, "ConcurrentTasks": 1,
+                "TaskTimeoutMinutes": settings["task_timeout_minutes"],
+                "OverrideJobFailureDetection": True, "FailureDetectionJobErrors": 3,
+                "ExtraInfoKeyValue0": "ComfyNativeProtocol=1", "ExtraInfoKeyValue1": "ComfyNativeRunId=" + submission["run_id"],
+                "ExtraInfoKeyValue2": "ComfyNativeManifest=" + str(directory / "submission.json")}
+            if submission.get("plugin_directory"):
+                job["CustomPluginDirectory"] = submission["plugin_directory"]
+            if settings["workers"]:
+                job["Whitelist"] = settings["workers"]
+            plugin = {"NativeQueueProtocol": 1, "NativeRunId": submission["run_id"], "NativeRoot": str(store.root),
+                "BatchCount": 1, "ChunkSize": 1}
+            for name, values in (("job.info", job), ("plugin.info", plugin)):
+                (directory / name).write_text("".join(field(k) + "=" + field(v) + "\n" for k, v in values.items()), encoding="utf-8")
+            receipt["phase"] = "submitting"
+            atomic_json(receipt_path, receipt)
+            output = self.command(str(directory / "job.info"), str(directory / "plugin.info"))
+            job_id = DeadlineCommandHelper.get_job_id_from_submission(output)
+            if not re.fullmatch(r"[0-9a-f]{24}", job_id):
+                raise RuntimeError("Deadline did not confirm a Job ID; the request remains unconfirmed")
+            receipt.update(job_id=job_id, task_id="0", phase="suspended")
+            atomic_json(receipt_path, receipt)
+        job_id = receipt["job_id"]
+        state = parse_ini(self.command("-GetJob", job_id, "True")).get(job_id, {}).get("Status")
+        if state is None:
+            return receipt  # refresh confirms absence before closing the run.
+        if state == "Suspended":
+            if store.control(submission["run_id"])["cancel_requested"]:
+                return receipt
+            self.command("-ResumeJob", job_id)
+            state = parse_ini(self.command("-GetJob", job_id, "True")).get(job_id, {}).get("Status")
+            if state is None:
+                return receipt
+        if state == "Failed":
+            receipt["phase"] = "failed"
+            atomic_json(receipt_path, receipt)
+            return receipt
+        if state not in {"Active", "Completed"}:
+            raise RuntimeError("Deadline activation is unconfirmed: " + state)
+        receipt.update(accepted=True, phase="accepted", accepted_at=int(time.time() * 1000))
+        atomic_json(receipt_path, receipt)
+        return receipt
+
+    def refresh(self):
+        with self.lock:
+            settings = dict(self.settings)
+            cached_ids = set(self.jobs)
+        if not settings["root"]:
+            return
+        store = RunStore(settings["root"])
+        store.check_root()
+        submissions = list(store.records(settings["scope"]))
+        mappings = {}
+        for submission in submissions:
+            directory = store.run(submission["run_id"])
+            if not (directory / "receipt.json").exists() or not read_json(directory / "receipt.json").get("accepted"):
+                with self.lock:
+                    recover = settings["enabled"] and self.settings["enabled"] and not self.configuration_in_progress and all(settings[k] == self.settings[k] for k in ("root", "scope"))
+                    if recover:
+                        self.active_submissions += 1
+                if recover:
+                    try:
+                        with record_lock(store.key(settings["scope"], submission["submission_key"])):
+                            with record_lock(directory):
+                                self._ensure_submitted(store, submission)
+                    finally:
+                        with self.lock:
+                            self.active_submissions -= 1
+            if (directory / "receipt.json").exists():
+                receipt = read_json(directory / "receipt.json")
+                if receipt.get("job_id"):
+                    mappings[receipt["job_id"]] = submission
+        remote = {}
+        for offset in range(0, len(mappings), 100):
+            remote.update(parse_ini(self.command("-GetJob", ",".join(list(mappings)[offset:offset + 100]), "True")))
+        absent = [job_id for job_id in mappings if job_id not in remote]
+        for offset in range(0, len(absent), 100):
+            output = self.command("-ExecuteScript", str(Path(__file__).parent / "scripts" / "deadline_job_status.py"), *absent[offset:offset + 100])
+            lines = [line[len("COMFY_NATIVE_JOBS="):] for line in output.splitlines() if line.startswith("COMFY_NATIVE_JOBS=")]
+            if len(lines) != 1:
+                raise RuntimeError("Deadline did not confirm the missing Jobs; cached runs are retained")
+            confirmed = json.loads(lines[0])
+            if set(confirmed) != set(absent[offset:offset + 100]) or any(not isinstance(record, dict) or "Status" not in record for record in confirmed.values()):
+                raise ValueError("Invalid Deadline Job confirmation")
+            remote.update(confirmed)
+        jobs, histories, assets, progress = {}, {}, {}, []
+        for submission in submissions:
+            run_id = submission["run_id"]
+            directory = store.run(run_id)
+            receipt = read_json(directory / "receipt.json") if (directory / "receipt.json").exists() else {}
+            control = store.control(run_id)
+            fields = remote.get(receipt.get("job_id"), {})
+            if fields.get("Status") in {"Deleted", "Missing"} and receipt.get("job_id"):
+                with record_lock(directory):
+                    receipt = read_json(directory / "receipt.json")
+                    if receipt.get("phase") != "removed":
+                        receipt["phase"] = "removed"
+                        atomic_json(directory / "receipt.json", receipt)
+                    current = store.control(run_id)
+                    if not current.get("result") and not current.get("deadline_removed"):
+                        current.update(generation=current["generation"] + 1, deadline_removed=True,
+                            status="cancelled" if current.get("cancel_requested") else "failed", finished_at=int(time.time() * 1000))
+                        if current["status"] == "failed":
+                            current["error"] = {"exception_type": "DeadlineJobRemoved", "exception_message":
+                                "The Job was removed from Deadline before a result was committed. Retry to submit a new run."}
+                        atomic_json(directory / "control.json", current)
+                control = store.control(run_id)
+            elif control.get("cancel_requested") and control["status"] not in {"cancelled", "completed"} and receipt.get("job_id") and fields:
+                if fields.get("Status") not in {"Suspended", "Failed", "Completed"}:
+                    self.command("-SuspendJob", receipt["job_id"])
+                task = self.command("-GetJobTask", receipt["job_id"], "0")
+                task_fields = dict(line.split("=", 1) for line in task.splitlines() if "=" in line)
+                task_stopped = task_fields.get("Status") in {"Pending", "Queued", "Completed", "Failed", "Suspended"}
+                if task_stopped and (control["generation"] == 0 or control.get("execution_mode") == "task-owned"):
+                    with record_lock(directory):
+                        current = store.control(run_id)
+                        if current.get("cancel_requested") and not current.get("result") and current["generation"] == control["generation"]:
+                            current.update(status="cancelled", finished_at=int(time.time() * 1000))
+                            atomic_json(directory / "control.json", current)
+                    control = store.control(run_id)
+            status = control["status"]
+            error = control.get("error")
+            result = None
+            if control.get("result"):
+                cache_key = (settings["root"], run_id, control["result"])
+                if cache_key not in self.result_cache:
+                    self.result_cache[cache_key] = store.result(run_id)
+                result = self.result_cache[cache_key]
+                status = "completed"
+                if fields.get("Status") in {"Failed", "Active", "Suspended"} and not control.get("cancel_requested"):
+                    # Result won before acknowledgement loss. No rerender is needed.
+                    self.command("-CompleteJob", receipt["job_id"])
+            elif fields.get("Status") in {"Failed", "Completed"} and status != "cancelled":
+                status = "failed"
+                error = error or {"exception_type": "DeadlineError", "exception_message":
+                    "Deadline " + fields["Status"] + " without a committed ComfyUI result. Inspect task reports."}
+            elif status == "failed" and fields.get("Status") == "Active":
+                status = "pending"  # Deadline infrastructure retries remain one run.
+            elif fields.get("Status") == "Suspended" and not control.get("cancel_requested"):
+                status = "pending"  # External suspension is diagnostic, never user cancellation.
+            outputs = copy.deepcopy(result["outputs"]) if result else {}
+            file_assets = []
+            if result:
+                for item in result["files"]:
+                    name = Path(item["path"]).name
+                    output = {"filename": name, "subfolder": "deadline-native/" + run_id + "/" + str(Path(item["path"]).parent).replace("\\", "/"), "type": "output"}
+                    file_assets.append({**output, "node_id": item["node_id"], "output_key": item["output_key"],
+                        "output_index": item["output_index"], "size": item["size"], "mime_type": mimetypes.guess_type(name)[0] or "application/octet-stream",
+                        "id": str(uuid.uuid5(uuid.UUID(run_id), item["path"])), "name": name, "hash": item["sha256"],
+                        "preview_url": "/view?" + urlencode(output)})
+                    self._cache_output(store, run_id, item)
+                    outputs[item["node_id"]][item["output_key"]][item["output_index"]] = output
+            files = [i for i in file_assets if i["mime_type"].startswith(("image/", "video/", "audio/"))]
+            job = {"id": run_id, "status": status, "create_time": submission["created_at"],
+                "workflow_id": submission["workflow_id"], "outputs_count": len(file_assets), "previewable_outputs_count": len(files),
+                "workflow": submission["payload"], "outputs": outputs,
+                "execution_start_time": control.get("started_at"), "execution_end_time": control.get("finished_at"),
+                "deadline": {"job_id": receipt.get("job_id"), "worker": control.get("worker"), "status": fields.get("Status", "Unconfirmed"),
+                    "accepted": receipt.get("accepted", False), "retry_of": submission.get("retry_of"), "cancel_requested": control.get("cancel_requested")}}
+            if files:
+                first = files[0]
+                job["preview_output"] = {k: first[k] for k in ("filename", "subfolder", "type")}
+                job["preview_output"].update(nodeId=first["node_id"], mediaType=first["output_key"])
+            messages = result.get("execution_status", {}).get("messages", []) if result else []
+            if status == "cancelled":
+                messages = [["execution_start", {"timestamp": control.get("started_at", submission["created_at"])}],
+                    ["execution_interrupted", {"prompt_id": run_id, "timestamp": control.get("finished_at", int(time.time() * 1000))}]]
+            if error and status == "failed":
+                detail = {"prompt_id": run_id, "node_id": "", "node_type": "Deadline", "traceback": [],
+                    "current_inputs": {}, "current_outputs": {}, **error}
+                detail["prompt_id"] = run_id
+                job["execution_error"] = detail
+                messages = [["execution_error", detail]]
+            job["execution_status"] = {"completed": status == "completed", "status_str": "success" if status == "completed" else "error", "messages": messages}
+            histories[run_id] = {"prompt": [0, run_id, submission["payload"]["prompt"], submission["payload"]["extra_data"], list(submission["payload"]["prompt"])],
+                "outputs": outputs, "status": job["execution_status"]}
+            assets[run_id] = file_assets
+            if not control.get("hidden"):
+                jobs[run_id] = job
+            event = {"run_id": run_id, "generation": control["generation"], "sequence": 0, "status": status, "nodes": {}}
+            attempt = control.get("attempt_id")
+            if attempt and (directory / "attempts" / attempt / "progress.json").exists():
+                sample = read_json(directory / "attempts" / attempt / "progress.json")
+                if sample.get("run_id") == run_id and sample.get("generation") == control["generation"]:
+                    event.update(sample)
+            # A terminal/control transition must outrank its last live sample;
+            # otherwise the browser's sequence fence retains a running preview.
+            event["sequence"] = event["sequence"] * 4 + (2 if status in TERMINAL else int(bool(control.get("cancel_requested"))))
+            event.update(status=status, cancel_requested=control.get("cancel_requested", False))
+            for node in event["nodes"].values():
+                node["prompt_id"] = run_id
+            if not control.get("hidden"):
+                progress.append(event)
+        with self.lock:
+            if any(self.settings[k] != settings[k] for k in ("root", "scope")):
+                return
+            for run_id, job in self.jobs.items():
+                if run_id not in jobs and run_id not in cached_ids:
+                    jobs[run_id] = job
+                    histories[run_id] = self.histories[run_id]
+                    assets[run_id] = self.assets[run_id]
+            self.jobs, self.histories, self.assets, self.progress = jobs, histories, assets, progress
+            self.health = {"connected": True, "message": "Deadline and shared records are available", "last_refresh": int(time.time() * 1000)}
+
+    def _cache_output(self, store, run_id, asset):
+        # Native Add to workflow writes an annotated [output] path into a
+        # LoadImage/Video/Audio widget. Materialize that namespace in the host's
+        # normal output root; the share and accepted manifest remain authoritative.
+        if self.output_root is None:
+            return
+        target = safe_path(self.output_root, "deadline-native/" + run_id + "/" + asset["path"])
+        if target.is_file() and target.stat().st_size == asset["size"]:
+            return
+        source = safe_path(store.run(run_id), asset["path"])
+        target.parent.mkdir(parents=True, exist_ok=True)
+        temporary = target.with_name("." + target.name + "." + uuid.uuid4().hex + ".tmp")
+        try:
+            shutil.copy2(source, temporary)
+            if self._hash_file(temporary) != asset["sha256"]:
+                raise ValueError("Output cache hash mismatch")
+            os.replace(temporary, target)
+        finally:
+            if temporary.exists():
+                temporary.unlink()
+
+    async def list_jobs(self):
+        with self.lock:
+            return copy.deepcopy(list(self.jobs.values()))
+
+    async def get_job(self, run_id):
+        with self.lock:
+            return copy.deepcopy(self.jobs.get(run_id))
+
+    async def get_assets(self, run_id):
+        with self.lock:
+            return copy.deepcopy(self.assets.get(run_id, []))
+
+    async def get_history(self, run_id):
+        with self.lock:
+            return copy.deepcopy(self.histories[run_id])
+
+    async def view(self, request):
+        def resolve():
+            parts = request.query.get("subfolder", "").split("/")
+            run_id = canonical_id(parts[1])
+            with self.lock:
+                if run_id not in self.jobs or self.jobs[run_id]["status"] != "completed":
+                    raise ValueError("No accepted result in this user scope")
+            relative = "/".join(parts[2:]) + "/" + request.query.get("filename", "")
+            store = RunStore(self.settings["root"])
+            result = store.result(run_id)
+            if not result or not any(asset["path"] == relative for asset in result["files"]):
+                raise ValueError("Asset is not in the accepted result manifest")
+            return safe_path(store.run(run_id), relative)
+        try:
+            path = await asyncio.to_thread(resolve)
+            return web.FileResponse(path)
+        except (ValueError, OSError, KeyError, IndexError):
+            return web.Response(status=404)
+
+    async def cancel(self, run_id):
+        with self.lock:
+            if self.jobs.get(run_id, {}).get("status") in TERMINAL:
+                return False
+        changed = await asyncio.to_thread(RunStore(self.settings["root"]).cancel, run_id)
+        if changed:
+            with self.lock:
+                job = self.jobs.get(run_id)
+                if job and job["status"] not in TERMINAL:
+                    job["deadline"]["cancel_requested"] = True
+                for event in self.progress:
+                    if event["run_id"] == run_id and event["status"] not in TERMINAL:
+                        event["cancel_requested"] = True
+                        event["sequence"] = event["sequence"] // 4 * 4 + 1
+        self.wakeup.set()
+        return changed
+
+    async def hide(self, run_id):
+        def perform():
+            store = RunStore(self.settings["root"])
+            with record_lock(store.run(run_id)):
+                state = store.control(run_id)
+                if state["status"] not in TERMINAL:
+                    with self.lock:
+                        if self.jobs.get(run_id, {}).get("status") not in TERMINAL:
+                            raise ValueError("Only terminal history can be hidden")
+                state["hidden"] = True
+                atomic_json(store.run(run_id) / "control.json", state)
+        await asyncio.to_thread(perform)
+        self.wakeup.set()
+
+    def retry(self, run_id, key):
+        if not self.settings["enabled"]:
+            raise ValueError("Native Deadline queue is disabled")
+        store = RunStore(self.settings["root"])
+        original = read_json(store.run(run_id) / "submission.json")
+        if original["scope"] != self.settings["scope"]:
+            raise ValueError("Run is outside this user scope")
+        with self.lock:
+            status = self.jobs.get(run_id, {}).get("status", store.control(run_id)["status"])
+        if status not in TERMINAL:
+            raise ValueError("Only terminal runs may be retried")
+        # Retained staged inputs permit retry when the workstation is gone.
+        body = {**copy.deepcopy(original["payload"]), "submission_key": key, "batch_id": original["batch_id"]}
+        return self.submit(body, captured=original["settings"], retry_of=run_id)
+
+
+def register_backend():
+    if os.environ.get("DEADLINE_SLAVE_NAME"):
+        return None
+    try:
+        import server
+        instance = server.PromptServer.instance
+    except (ImportError, AttributeError):
+        return None
+    if instance is None:
+        return None
+    backend = FarmBackend()
+    try:
+        import comfyui_version
+        version = comfyui_version.__version__
+    except (ImportError, AttributeError):
+        version = "unknown"
+    supported = version == "0.39.0"
+    if supported:
+        instance.app.middlewares.append(NativeJobsAdapter(backend).middleware)
+
+    @instance.routes.get("/deadline/farm/capabilities")
+    async def capabilities(_request):
+        with backend.lock:
+            return web.json_response({"protocol": SCHEMA, "ready": backend.readiness_cache["ready"] and supported, "mode": "native" if supported else "unsupported",
+                "frontend_version": "1.53.10", "transport": "deadlinecommand",
+                "health": backend.health, "message": backend.health["message"] if supported else "Native queue requires ComfyUI 0.39.0; found " + version})
+
+    @instance.routes.get("/deadline/farm/settings")
+    async def settings(_request):
+        with backend.lock:
+            return web.json_response({"settings": backend.settings, "health": backend.health, "readiness": backend.readiness_cache})
+
+    async def action(request, callback):
+        try:
+            return web.json_response(await asyncio.to_thread(callback, await request.json()))
+        except SettingsConflict as exc:
+            return web.json_response({"error": {"type": "deadline_settings_conflict", "message": str(exc)}, "node_errors": {}}, status=409)
+        except (ValueError, TypeError, KeyError) as exc:
+            return web.json_response({"error": {"type": "deadline_validation_error", "message": str(exc)}, "node_errors": {}}, status=400)
+        except Exception as exc:
+            return web.json_response({"error": {"type": "deadline_unconfirmed", "message": str(exc)}, "node_errors": {}}, status=503)
+
+    @instance.routes.post("/deadline/farm/settings")
+    async def configure(request):
+        return await action(request, lambda body: backend.configure(body["settings"], body["expected_settings"]) if "expected_settings" in body else backend.configure(body))
+
+    @instance.routes.get("/deadline/farm/discovery")
+    async def discovery(request):
+        return web.json_response(await asyncio.to_thread(backend.discover, request.query.get("refresh") == "1"))
+
+    @instance.routes.get("/deadline/farm/readiness")
+    async def readiness(request):
+        return web.json_response(await asyncio.to_thread(backend.readiness, force=request.query.get("refresh") == "1"))
+
+    @instance.routes.get("/deadline/farm/setup-guide")
+    async def setup_guide(_request):
+        return web.Response(text=(Path(__file__).parent / "docs" / "DEADLINE_NATIVE_QUEUE_ADMIN.md").read_text(encoding="utf-8"), content_type="text/plain")
+
+    @instance.routes.post("/deadline/farm/submit")
+    async def submit(request):
+        if not supported:
+            return web.json_response({"error": "Unsupported ComfyUI version"}, status=409)
+        return await action(request, backend.submit)
+
+    @instance.routes.post("/deadline/farm/{run_id}/retry")
+    async def retry(request):
+        if not supported:
+            return web.json_response({"error": "Unsupported ComfyUI version"}, status=409)
+        return await action(request, lambda body: backend.retry(canonical_id(request.match_info["run_id"]), body["submission_key"]))
+
+    @instance.routes.get("/deadline/farm/progress")
+    async def progress(_request):
+        with backend.lock:
+            return web.json_response({"runs": backend.progress, "health": backend.health, "readiness": backend.readiness_cache})
+
+    @instance.routes.get("/deadline/farm/{run_id}/preview")
+    async def preview(request):
+        def resolve():
+            run_id = canonical_id(request.match_info["run_id"])
+            with backend.lock:
+                if run_id not in backend.jobs:
+                    raise ValueError("Run is outside this user scope")
+            store = RunStore(backend.settings["root"])
+            state = store.control(run_id)
+            if str(state["generation"]) != request.query.get("generation") or not state.get("attempt_id"):
+                raise ValueError("Preview is from an old generation")
+            event = read_json(store.run(run_id) / "attempts" / state["attempt_id"] / "progress.json")
+            return safe_path(store.run(run_id), event["preview_file"])
+        try:
+            return web.FileResponse(await asyncio.to_thread(resolve))
+        except (ValueError, OSError, KeyError):
+            return web.Response(status=404)
+
+    async def close(_app):
+        backend.shutdown.set()
+        backend.wakeup.set()
+        if backend.thread:
+            await asyncio.to_thread(backend.thread.join, 35)
+    instance.app.on_cleanup.append(close)
+    if supported:
+        backend.start()
+    return backend

farm_protocol.py

@@ -0,0 +1,288 @@
+"""Version 1 farm records. Shared verbatim by ComfyUI and the Deadline sandbox.
+
+Windows byte-range locks and same-directory os.replace were exercised on SMB
+from M28 and SIM01. No registry depends on the submitting process remaining up.
+"""
+import contextlib
+import hashlib
+import json
+import os
+import tempfile
+import time
+import uuid
+from pathlib import Path
+
+if os.name == "nt":
+    import ctypes
+    import msvcrt
+    from ctypes import wintypes
+
+    _kernel = ctypes.WinDLL("kernel32", use_last_error=True)
+    _kernel.CreateFileW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD,
+        ctypes.c_void_p, wintypes.DWORD, wintypes.DWORD, wintypes.HANDLE]
+    _kernel.CreateFileW.restype = wintypes.HANDLE
+    _kernel.CloseHandle.argtypes = [wintypes.HANDLE]
+    _kernel.ReadFile.argtypes = [wintypes.HANDLE, ctypes.c_void_p, wintypes.DWORD,
+        ctypes.POINTER(wintypes.DWORD), ctypes.c_void_p]
+    _kernel.VirtualAlloc.argtypes = [ctypes.c_void_p, ctypes.c_size_t, wintypes.DWORD, wintypes.DWORD]
+    _kernel.VirtualAlloc.restype = ctypes.c_void_p
+    _kernel.VirtualFree.argtypes = [ctypes.c_void_p, ctypes.c_size_t, wintypes.DWORD]
+
+SCHEMA = 1
+TERMINAL = {"completed", "failed", "cancelled"}
+
+
+def canonical_id(value):
+    if not isinstance(value, str) or str(uuid.UUID(value)) != value:
+        raise ValueError("Expected a canonical lowercase UUID")
+    return value
+
+
+def digest(value):
+    return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"),
+        ensure_ascii=False, allow_nan=False).encode("utf-8")).hexdigest()
+
+
+def safe_path(root, relative):
+    root = Path(root).resolve()
+    if not isinstance(relative, str) or not relative or Path(relative).is_absolute() or ":" in relative:
+        raise ValueError("Expected a relative asset path")
+    path = (root / relative).resolve()
+    if os.path.commonpath([str(root), str(path)]) != str(root) or path == root:
+        raise ValueError("Asset path escapes its root")
+    return path
+
+
+def _read_unc(path):
+    # The NAS can retain a read handle to an atomically replaced file. Open
+    # briefly without sharing, with DELETE access and no data caching, so a
+    # replacement breaks the old handle lease. No delete is performed.
+    native = _kernel.CreateFileW(str(path), 0x80010000, 0, None, 3, 0x20000080, None)
+    if native == ctypes.c_void_p(-1).value:
+        raise ctypes.WinError(ctypes.get_last_error())
+    # Unbuffered reads require aligned addresses and lengths. VirtualAlloc
+    # supplies a page-aligned 64 KiB buffer; EOF may return a shorter record.
+    buffer = _kernel.VirtualAlloc(None, 65536, 0x3000, 4)
+    if not buffer:
+        _kernel.CloseHandle(native)
+        raise ctypes.WinError(ctypes.get_last_error())
+    chunks = []
+    try:
+        while True:
+            count = wintypes.DWORD()
+            if not _kernel.ReadFile(native, buffer, 65536, ctypes.byref(count), None):
+                error = ctypes.get_last_error()
+                if error == 38:  # ERROR_HANDLE_EOF
+                    break
+                raise ctypes.WinError(error)
+            chunks.append(ctypes.string_at(buffer, count.value))
+            if count.value < 65536:
+                break
+        return b"".join(chunks).decode("utf-8-sig")
+    finally:
+        _kernel.CloseHandle(native)
+        _kernel.VirtualFree(buffer, 0, 0x8000)
+
+
+def read_json(path):
+    path = Path(path)
+    deadline = time.monotonic() + 2
+    while True:
+        try:
+            if os.name == "nt" and str(path).startswith("\\\\"):
+                return json.loads(_read_unc(path))
+            with path.open(encoding="utf-8-sig") as handle:
+                return json.load(handle)
+        except PermissionError:
+            if time.monotonic() >= deadline:
+                raise
+            time.sleep(.01)
+
+
+def atomic_json(path, value):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix="publish-", suffix=".tmp", dir=str(path.parent))
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as handle:
+            json.dump(value, handle, ensure_ascii=False, allow_nan=False)
+            handle.flush()
+            os.fsync(handle.fileno())
+        deadline = time.monotonic() + 2
+        while True:
+            try:
+                os.replace(temporary, path)
+                break
+            except PermissionError:
+                if time.monotonic() >= deadline:
+                    raise
+                time.sleep(.01)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+@contextlib.contextmanager
+def record_lock(directory, timeout=30):
+    directory = Path(directory)
+    directory.mkdir(parents=True, exist_ok=True)
+    with (directory / "control.lock").open("a+b") as handle:
+        deadline = time.monotonic() + timeout
+        while True:
+            try:
+                handle.seek(0)
+                if os.name == "nt":
+                    import msvcrt
+                    msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+                else:
+                    import fcntl
+                    fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except OSError:
+                if time.monotonic() >= deadline:
+                    raise TimeoutError("Could not acquire farm record lock: " + str(directory)) from None
+                time.sleep(.025)
+        try:
+            yield
+        finally:
+            handle.seek(0)
+            if os.name == "nt":
+                msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+            else:
+                fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+
+
+class RunStore:
+    def __init__(self, root):
+        self.root = Path(root)
+
+    def initialize(self):
+        with record_lock(self.root):
+            path = self.root / "store.json"
+            if not path.exists():
+                atomic_json(path, {"schema": SCHEMA, "store_id": str(uuid.uuid4())})
+            self.check_root()
+
+    def check_root(self):
+        try:
+            identity = read_json(self.root / "store.json")
+        except FileNotFoundError:
+            raise RuntimeError("Farm storage identity is unavailable; reconnect the configured share before submitting") from None
+        if identity.get("schema") != SCHEMA:
+            raise ValueError("Unsupported farm storage schema")
+        canonical_id(identity["store_id"])
+        return identity
+
+    def run(self, run_id):
+        return self.root / "runs" / canonical_id(run_id)
+
+    def key(self, scope, submission_key):
+        return self.root / "requests" / digest(scope)[:24] / canonical_id(submission_key)
+
+    def records(self, scope):
+        for path in (self.root / "runs").glob("*/submission.json"):
+            submission = read_json(path)
+            if submission.get("schema") != SCHEMA:
+                raise ValueError("Unsupported farm record schema: " + str(path))
+            if submission.get("scope") == scope:
+                yield submission
+
+    def control(self, run_id):
+        return read_json(self.run(run_id) / "control.json")
+
+    def begin_attempt(self, run_id, attempt_id, job_id, worker, assignment, execution_mode=None):
+        directory = self.run(run_id)
+        canonical_id(attempt_id)
+        with record_lock(directory):
+            receipt = read_json(directory / "receipt.json")
+            if receipt["job_id"] != job_id:
+                raise ValueError("Deadline assignment does not belong to this run")
+            state = self.control(run_id)
+            if state.get("result") or state.get("cancel_requested") or state.get("deadline_removed"):
+                return None
+            # Assignment is a Worker plus Deadline task-start timestamp. An old
+            # process cannot reclaim a generation after a newer assignment.
+            if state.get("assignment") == assignment:
+                if state.get("attempt_id") != attempt_id:
+                    raise ValueError("This Deadline task assignment already has an attempt")
+                return state
+            prior_start = state.get("assignment_start", 0)
+            if assignment["start"] <= prior_start:
+                raise ValueError("Stale Deadline task assignment")
+            state.update(generation=state["generation"] + 1, attempt_id=attempt_id,
+                worker=worker, assignment=assignment, assignment_start=assignment["start"],
+                status="in_progress", started_at=int(time.time() * 1000), error=None)
+            state.pop("execution_mode", None)
+            if execution_mode is not None:
+                state["execution_mode"] = execution_mode
+            atomic_json(directory / "control.json", state)
+            return state
+
+    def owns(self, state, attempt_id, generation):
+        return state.get("attempt_id") == attempt_id and state["generation"] == generation
+
+    def update_attempt(self, run_id, attempt_id, generation, **values):
+        directory = self.run(run_id)
+        with record_lock(directory):
+            state = self.control(run_id)
+            if not self.owns(state, attempt_id, generation) or state.get("result") or state.get("status") == "cancelled":
+                return False
+            state.update(values)
+            atomic_json(directory / "control.json", state)
+            return True
+
+    def cancel(self, run_id):
+        directory = self.run(run_id)
+        with record_lock(directory):
+            state = self.control(run_id)
+            if state.get("result") or state.get("status") in {"completed", "cancelled"} or state.get("deadline_removed"):
+                return False
+            state.update(cancel_requested=True, cancel_requested_at=int(time.time() * 1000))
+            atomic_json(directory / "control.json", state)
+            return True
+
+    def commit(self, run_id, attempt_id, generation, manifest, assignment_is_current=None):
+        directory = self.run(run_id)
+        if manifest.get("run_id") != run_id or manifest.get("attempt_id") != attempt_id or manifest.get("generation") != generation:
+            raise ValueError("Result identity mismatch")
+        # Complete files precede the immutable manifest, which precedes pointer.
+        result_path = directory / "attempts" / canonical_id(attempt_id) / "result.json"
+        with record_lock(directory):
+            state = self.control(run_id)
+            if state.get("result"):
+                return state["result"] == str(result_path.relative_to(directory)).replace("\\", "/")
+            if not self.owns(state, attempt_id, generation) or state.get("cancel_requested"):
+                return False
+            if assignment_is_current is not None and not assignment_is_current():
+                return False
+            self.validate_result(run_id, manifest)
+            atomic_json(result_path, manifest)
+            state.update(result=str(result_path.relative_to(directory)).replace("\\", "/"),
+                status="completed", finished_at=int(time.time() * 1000))
+            atomic_json(directory / "control.json", state)
+            return True
+
+    def validate_result(self, run_id, manifest):
+        if manifest.get("schema") != SCHEMA or not manifest.get("files"):
+            raise ValueError("Result has no validated files or unsupported schema")
+        prefix = "attempts/" + canonical_id(manifest["attempt_id"]) + "/"
+        for asset in manifest["files"]:
+            if not asset["path"].startswith(prefix):
+                raise ValueError("Result file is outside its attempt")
+            path = safe_path(self.run(run_id), asset["path"])
+            if not path.is_file() or path.stat().st_size != asset["size"] or asset["size"] <= 0:
+                raise ValueError("Missing or invalid published output: " + str(path))
+            with path.open("rb") as handle:
+                checksum = hashlib.file_digest(handle, "sha256").hexdigest() if hasattr(hashlib, "file_digest") else hashlib.sha256(handle.read()).hexdigest()
+            if checksum != asset["sha256"]:
+                raise ValueError("Published output hash mismatch: " + str(path))
+
+    def result(self, run_id):
+        state = self.control(run_id)
+        if not state.get("result"):
+            return None
+        manifest = read_json(safe_path(self.run(run_id), state["result"]))
+        if manifest.get("run_id") != run_id or not self.owns(state, manifest.get("attempt_id"), manifest.get("generation")):
+            raise ValueError("Accepted result identity mismatch")
+        self.validate_result(run_id, manifest)
+        return manifest

farm_readiness.py

@@ -0,0 +1,74 @@
+"""Submission readiness from saved settings and a read-only farm snapshot."""
+import os
+import re
+
+
+def off_host_root(value):
+    if not isinstance(value, str) or not re.match(r"^\\\\[^\\/]+\\[^\\/]+", value):
+        return False
+    host = value.split("\\")[2].lower()
+    return host not in {"localhost", "127.0.0.1", "::1", os.environ.get("COMPUTERNAME", "").lower()}
+
+
+def readiness(settings, snapshot):
+    checks = []
+    def check(identity, label, ok, message):
+        checks.append({"id": identity, "label": label, "ok": bool(ok), "message": message})
+
+    check("enabled", "Farm submission", settings.get("enabled") is True,
+        "Farm submission is enabled." if settings.get("enabled") else "The farm queue is disabled in the host configuration.")
+    storage = snapshot.get("storage", {})
+    root_ok = off_host_root(settings.get("root")) and bool(str(settings.get("scope", "")).strip())
+    check("storage", "Shared storage", root_ok and storage.get("connected"),
+        storage.get("message", "Shared storage has not been checked.") if root_ok else "Choose an off-host shared folder and a user scope.")
+    check("repository", "Deadline repository", snapshot.get("connected"),
+        snapshot.get("message", "Deadline repository has not been checked."))
+    check("plugin", "Native Deadline plugin", snapshot.get("plugin_available"),
+        "Native plugin files are installed." if snapshot.get("plugin_available") else "Install the native ComfyUI plugin before submitting.")
+    if snapshot.get("configuration_error"):
+        check("session_configuration", "Worker session configuration", False, snapshot["configuration_error"])
+
+    known = {worker["name"].lower(): worker for worker in snapshot.get("workers", [])}
+    selected = {}
+    for name in str(settings.get("workers", "")).split(","):
+        if name.strip():
+            selected.setdefault(name.strip().lower(), name.strip())
+    workers = []
+    for key, name in selected.items():
+        worker = known.get(key)
+        reasons = []
+        if worker is None:
+            reasons.append("not found in the repository")
+        else:
+            name = worker["name"]
+            if not worker.get("enabled"):
+                reasons.append("disabled in Deadline")
+            mode = str(worker.get("scheduling_mode", "AllJobs")).strip().lower()
+            scope = str(snapshot.get("deadline_user", settings.get("scope", ""))).strip()
+            if mode == "userjobs":
+                users = worker.get("user_job_names", [])
+                if scope.lower() not in {str(user).strip().lower() for user in users}:
+                    reasons.append("does not accept Deadline user " + scope + " (allowed: " + (", ".join(users) or "no users") + ")")
+            elif mode == "machinejobs":
+                machine_user = str(worker.get("machine_user", "")).strip()
+                if not machine_user:
+                    reasons.append("Deadline machine user could not be verified")
+                elif scope.lower() != machine_user.lower():
+                    reasons.append("accepts only Deadline machine user " + machine_user)
+            elif mode != "alljobs":
+                reasons.append("Deadline scheduling mode " + str(worker.get("scheduling_mode")) + " could not be verified")
+            for field in ("pool", "group"):
+                value = settings.get(field, "none")
+                if value.lower() != "none" and value.lower() not in {str(item).lower() for item in worker.get(field + "s", [])}:
+                    reasons.append("outside selected " + field)
+            if worker.get("session_error"):
+                reasons.append(worker["session_error"])
+        workers.append({"name": name, "eligible": not reasons, "status": worker.get("status") if worker else "Missing", "reasons": reasons})
+    invalid = [worker["name"] + ": " + ", ".join(worker["reasons"]) for worker in workers if not worker["eligible"]]
+    worker_message = "; ".join(invalid) if invalid else f"{len(workers)} selected Worker(s) can receive farm jobs."
+    if not workers:
+        worker_message = "Deadline chooses a Worker from the selected pool and group."
+    check("workers", "Worker restriction", not invalid, worker_message)
+    failed = next((item for item in checks if not item["ok"]), None)
+    return {"ready": failed is None, "code": failed["id"] if failed else "ready",
+        "message": failed["message"] if failed else "Ready to submit to Deadline.", "checks": checks, "workers": workers}

farm_session_owner.py

@@ -0,0 +1,494 @@
+"""Worker-local session lifetime only. Install independently of Deadline tasks.
+
+Owned children are tied to a Windows kill-on-close job object. Borrowed artist
+processes are never adopted by that object or terminated. Scheduling is Deadline's.
+"""
+
+import argparse
+import ctypes
+from ctypes import wintypes
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from urllib.parse import urlsplit
+import uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from farm_protocol import atomic_json, read_json
+
+
+class BasicLimits(ctypes.Structure):
+    _fields_ = [("process_time", ctypes.c_int64), ("job_time", ctypes.c_int64),
+        ("flags", wintypes.DWORD), ("minimum", ctypes.c_size_t), ("maximum", ctypes.c_size_t),
+        ("process_limit", wintypes.DWORD), ("affinity", ctypes.c_size_t),
+        ("priority", wintypes.DWORD), ("scheduling", wintypes.DWORD)]
+
+
+class ExtendedLimits(ctypes.Structure):
+    _fields_ = [("basic", BasicLimits), ("io", ctypes.c_uint64 * 6),
+        ("process_memory", ctypes.c_size_t), ("job_memory", ctypes.c_size_t),
+        ("peak_process", ctypes.c_size_t), ("peak_job", ctypes.c_size_t)]
+
+
+class OwnedProcess:
+    def __init__(self, command, cwd, log):
+        self.kernel = ctypes.WinDLL("kernel32", use_last_error=True)
+        self.kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR]
+        self.kernel.CreateJobObjectW.restype = wintypes.HANDLE
+        self.kernel.SetInformationJobObject.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
+        self.kernel.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE]
+        self.kernel.CloseHandle.argtypes = [wintypes.HANDLE]
+        self.handle = self.kernel.CreateJobObjectW(None, None)
+        if not self.handle:
+            raise ctypes.WinError(ctypes.get_last_error())
+        limits = ExtendedLimits()
+        limits.basic.flags = 0x2000  # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+        if not self.kernel.SetInformationJobObject(self.handle, 9, ctypes.byref(limits), ctypes.sizeof(limits)):
+            self.kernel.CloseHandle(self.handle)
+            raise ctypes.WinError(ctypes.get_last_error())
+        self.process = subprocess.Popen(command, cwd=cwd, stdout=log, stderr=subprocess.STDOUT,
+            creationflags=subprocess.CREATE_NO_WINDOW | 4)  # CREATE_SUSPENDED
+        if not self.kernel.AssignProcessToJobObject(self.handle, wintypes.HANDLE(int(self.process._handle))):
+            self.process.kill()
+            self.process.wait()
+            self.kernel.CloseHandle(self.handle)
+            raise ctypes.WinError(ctypes.get_last_error())
+        ntdll = ctypes.WinDLL("ntdll")
+        ntdll.NtResumeProcess.argtypes = [wintypes.HANDLE]
+        if ntdll.NtResumeProcess(wintypes.HANDLE(int(self.process._handle))) != 0:
+            self.close()
+            raise RuntimeError("Cannot resume supervised ComfyUI process")
+
+    def close(self):
+        self.kernel.CloseHandle(self.handle)
+        self.process.wait(timeout=15)
+
+
+def comfy_request(url, body=None):
+    request = urllib.request.Request(url, data=json.dumps(body).encode() if body is not None else None,
+        headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(request, timeout=5) as response:
+        raw = response.read()
+        return json.loads(raw) if raw else {}
+
+
+class SessionOwner:
+    def __init__(self, args):
+        self.args = args
+        self.lock = threading.RLock()
+        self.owned = None
+        self.log = None
+        self.session_id = None
+        self.endpoint_identity = None
+        self.reservation = None
+        self.last_active = time.monotonic()
+        self.draining = False
+        self.cold_start_seconds = None
+        self.url = args.borrowed_url or f"http://127.0.0.1:{args.comfy_port}"
+        self.borrowed = bool(args.borrowed_url)
+        if self.borrowed:
+            address = urlsplit(args.borrowed_url)
+            if address.scheme != "http" or address.hostname not in {"127.0.0.1", "localhost", "::1"} or address.username or address.password or address.path not in {"", "/"} or address.query or address.fragment:
+                raise ValueError("Borrowed sessions must use a worker-local HTTP endpoint")
+            self.url = args.borrowed_url.rstrip("/")
+        self.progress = {}
+        self.preview = None
+        self.events_stop = threading.Event()
+        self.events_ready = threading.Event()
+        args.state_root.mkdir(parents=True, exist_ok=True)
+        self.input_root = args.state_root / "input"
+        self.output_root = args.state_root / "output"
+        self.input_root.mkdir(exist_ok=True)
+        self.output_root.mkdir(exist_ok=True)
+        self.temp_root = args.state_root / "temp" / "temp"
+        self.temp_root.mkdir(parents=True, exist_ok=True)
+        if self.borrowed:
+            if not args.borrowed_input_root or not args.borrowed_output_root:
+                raise ValueError("Borrowed sessions require their real input and output roots")
+            self.input_root, self.output_root = args.borrowed_input_root, args.borrowed_output_root
+            self.temp_root = args.borrowed_temp_root or self.temp_root
+            path = args.state_root / "reservation.json"
+            if path.exists():
+                prior = read_json(path)
+                if prior.get("reservation"):
+                    current = comfy_request(self.url + "/deadline/session")
+                    if any(current.get(k) != prior["endpoint_identity"].get(k) for k in ("pid", "session_id", "comfyui_root")):
+                        raise ValueError("Borrowed endpoint changed while a reservation was retained; reconcile its old prompt")
+                    self.endpoint_identity, self.session_id = current, current["session_id"]
+                    self.reservation = prior["reservation"]
+                    self.reservation["last_client_at"] = time.monotonic() - 16
+
+    def persist_reservation(self):
+        atomic_json(self.args.state_root / "reservation.json", {"endpoint_identity": self.endpoint_identity,
+            "reservation": self.reservation})
+
+    def clear_reservation(self):
+        token = (self.reservation or {}).get("token")
+        self.reservation = None
+        self.persist_reservation()
+        self.events_stop.set()
+        self.preview = None
+        if token:
+            for suffix in (".png", ".jpg", ".tmp"):
+                try:
+                    (self.args.state_root / ("preview-" + token + suffix)).unlink(missing_ok=True)
+                except OSError:
+                    pass  # A preview reader must not prevent GPU handoff.
+        self.last_active = time.monotonic()
+
+    def assignment_active(self):
+        """Unknown/unavailable is never authority to stop a reservation."""
+        command = Path(os.environ.get("DEADLINE_PATH", r"C:\Program Files\Thinkbox\Deadline10\bin")) / "deadlinecommand.exe"
+        try:
+            checked = subprocess.run([str(command), "-GetJobTask", self.reservation["job_id"], "0"],
+                capture_output=True, text=True, timeout=10, creationflags=subprocess.CREATE_NO_WINDOW)
+        except (OSError, subprocess.SubprocessError):
+            return None
+        if checked.returncode or any(line.startswith("Error:") for line in checked.stdout.splitlines()):
+            return None
+        fields = dict(line.split("=", 1) for line in checked.stdout.splitlines() if "=" in line)
+        if "Status" not in fields or "SlaveName" not in fields:
+            return None
+        return fields["SlaveName"] == self.args.worker and fields["Status"] == "Rendering"
+
+    def identity(self):
+        return {"protocol": 1, "mode": "borrowed" if self.borrowed else "owned", "worker": self.args.worker, "gpu_uuid": self.args.gpu_uuid,
+            "comfyui_root": str(self.args.comfy_root), "owner_pid": os.getpid(),
+            "pid": self.owned.process.pid if self.owned else (self.endpoint_identity or {}).get("pid"), "session_id": self.session_id,
+            "endpoint_identity": self.endpoint_identity, "draining": self.draining,
+            "reservation": self.reservation, "cold_start_seconds": self.cold_start_seconds,
+            "url": self.url, "input_root": str(self.input_root), "output_root": str(self.output_root),
+            "temp_root": str(self.temp_root), "exclusive_worker": self.args.exclusive_worker}
+
+    def start(self):
+        if self.borrowed:
+            identity = comfy_request(self.url + "/deadline/session")
+            if Path(identity["comfyui_root"]).resolve() != self.args.comfy_root.resolve():
+                raise ValueError("Borrowed endpoint installation does not match")
+            gpu_lines = subprocess.check_output(["nvidia-smi", "--query-compute-apps=pid,gpu_uuid", "--format=csv,noheader"], text=True)
+            if not any(line.split(",")[0].strip() == str(identity["pid"]) and line.split(",")[1].strip() == self.args.gpu_uuid for line in gpu_lines.splitlines()):
+                raise ValueError("Cannot prove borrowed endpoint PID/GPU UUID; use a configured owned session")
+            self.endpoint_identity, self.session_id = identity, identity["session_id"]
+            return
+        try:
+            comfy_request(self.url + "/system_stats")
+        except (urllib.error.URLError, TimeoutError):
+            pass
+        else:
+            raise ValueError("Test port is occupied; an existing artist endpoint will not be adopted or killed")
+        gpu_lines = subprocess.check_output(["nvidia-smi", "--query-gpu=index,uuid", "--format=csv,noheader"], text=True)
+        indices = {line.split(",")[1].strip(): line.split(",")[0].strip() for line in gpu_lines.splitlines()}
+        if self.args.gpu_uuid not in indices:
+            raise ValueError("GPU UUID is not present on this Worker")
+        python = self.args.comfy_root.parent / "python_embeded" / "python.exe"
+        command = [str(python), "-s", str(self.args.comfy_root / "main.py"), "--port", str(self.args.comfy_port),
+            "--cuda-device", indices[self.args.gpu_uuid], "--disable-auto-launch",
+            "--preview-method", getattr(self.args, "preview_method", "auto"),
+            "--input-directory", str(self.input_root), "--output-directory", str(self.output_root),
+            "--temp-directory", str(self.temp_root.parent)]
+        if self.args.custom_nodes:
+            command += ["--disable-all-custom-nodes", "--whitelist-custom-nodes", *self.args.custom_nodes]
+        command += self.args.extra_args
+        self.log = (self.args.state_root / "comfy.log").open("ab", buffering=0)
+        self.owned = OwnedProcess(command, str(self.args.comfy_root), self.log)
+        started = time.monotonic()
+        deadline = started + getattr(self.args, "startup_seconds", 120)
+        while time.monotonic() < deadline:
+            if self.owned.process.poll() is not None:
+                raise RuntimeError("Owned ComfyUI exited at startup; inspect comfy.log")
+            try:
+                identity = comfy_request(self.url + "/deadline/session")
+            except urllib.error.HTTPError as exc:
+                if exc.code == 404:
+                    raise RuntimeError("Worker ComfyUI installation lacks ComfyUI-Deadline-Plugin (/deadline/session returned 404)") from exc
+                raise
+            except (urllib.error.URLError, TimeoutError):
+                time.sleep(0.25)
+                continue
+            if identity["pid"] != self.owned.process.pid or Path(identity["comfyui_root"]).resolve() != self.args.comfy_root.resolve():
+                raise RuntimeError("Owned endpoint PID/root did not match the launched process")
+            self.endpoint_identity = identity
+            self.session_id = str(uuid.uuid4())
+            self.cold_start_seconds = time.monotonic() - started
+            return
+        raise TimeoutError("Owned ComfyUI startup timed out")
+
+    def reserve(self, body):
+        with self.lock:
+            if body.get("worker") != self.args.worker or body.get("gpu_uuid") != self.args.gpu_uuid:
+                raise ValueError("Reservation Worker/GPU identity mismatch")
+            if self.draining or self.reservation:
+                raise ValueError("Session is draining or already reserved")
+            if self.owned and self.owned.process.poll() is not None:
+                self.stop()
+            if self.owned is None and (not self.borrowed or self.endpoint_identity is None):
+                try:
+                    self.start()
+                except Exception:
+                    self.stop()
+                    raise
+            self.reservation = {"token": str(uuid.uuid4()), "run_id": str(uuid.UUID(body["run_id"])),
+                "attempt_id": str(uuid.UUID(body["attempt_id"])), "job_id": body.get("job_id"),
+                "prompt_id": None, "reserved_at": time.time(), "last_client_at": time.monotonic(),
+                "client_id": str(uuid.uuid4())}
+            self.persist_reservation()
+            self.progress, self.preview = {}, None
+            self.events_stop = threading.Event()
+            self.events_ready = threading.Event()
+            threading.Thread(target=self.read_events, args=(self.reservation["token"], self.reservation["client_id"], self.events_stop), daemon=True).start()
+            self.events_ready.wait(2)
+            self.last_active = time.monotonic()
+            return self.identity()
+
+    def queue(self, body):
+        with self.lock:
+            self.verify_token(body)
+            if self.reservation["prompt_id"]:
+                return {"prompt_id": self.reservation["prompt_id"]}
+            self.check_identity()
+            self.reservation["prompt_id"] = str(uuid.uuid4())
+            self.persist_reservation()  # Recover exact prompt after a lost response/owner death.
+            payload = dict(body["payload"], client_id=self.reservation["client_id"], prompt_id=self.reservation["prompt_id"])
+            result = comfy_request(self.url + "/prompt", payload)
+            if result["prompt_id"] != self.reservation["prompt_id"]:
+                raise ValueError("Worker did not preserve the reserved prompt UUID")
+            return result
+
+    def verify_token(self, body):
+        if self.reservation is None or body.get("token") != self.reservation["token"]:
+            raise ValueError("Reservation token does not own this session")
+        self.reservation["last_client_at"] = time.monotonic()
+
+    def check_identity(self):
+        identity = comfy_request(self.url + "/deadline/session")
+        if not self.endpoint_identity or any(identity.get(k) != self.endpoint_identity.get(k) for k in ("pid", "session_id", "comfyui_root")):
+            raise ValueError("Worker endpoint identity changed; prompt was not dispatched")
+
+    def read_events(self, token, client_id, stop):
+        import asyncio
+        import aiohttp
+        async def read():
+            async with aiohttp.ClientSession() as client:
+                async with client.ws_connect(self.url.replace("http", "ws", 1) + "/ws?clientId=" + client_id, heartbeat=15) as socket:
+                    self.events_ready.set()
+                    while not stop.is_set():
+                        try:
+                            message = await socket.receive(timeout=1)
+                        except asyncio.TimeoutError:
+                            continue
+                        with self.lock:
+                            if self.reservation is None or self.reservation["token"] != token:
+                                return
+                            if message.type == aiohttp.WSMsgType.TEXT:
+                                event = json.loads(message.data)
+                                data = event.get("data", {})
+                                if data.get("prompt_id") != self.reservation["prompt_id"]:
+                                    continue
+                                if event["type"] == "progress" and data.get("node"):
+                                    node = str(data["node"])
+                                    self.progress[node] = {"node_id": node, "prompt_id": self.reservation["run_id"],
+                                        "value": data["value"], "max": data["max"], "state": "running"}
+                                elif event["type"] == "progress_state":
+                                    self.progress = data.get("nodes", {})
+                                elif event["type"] == "executing" and data.get("node"):
+                                    node = str(data["node"])
+                                    self.progress[node] = {"node_id": node, "prompt_id": self.reservation["run_id"], "value": 0, "max": 1, "state": "running"}
+                            elif message.type == aiohttp.WSMsgType.BINARY and 8 < len(message.data) <= 4 * 1024 ** 2:
+                                # Legacy previews arrive only on this reservation's
+                                # client socket. They never touch the browser canvas.
+                                raw = message.data
+                                image_bytes = None
+                                if int.from_bytes(raw[:4], "big") == 1:
+                                    extension = ".jpg" if int.from_bytes(raw[4:8], "big") == 1 else ".png"
+                                    image_bytes = raw[8:]
+                                elif int.from_bytes(raw[:4], "big") == 4:
+                                    length = int.from_bytes(raw[4:8], "big")
+                                    if 0 < length < len(raw) - 8:
+                                        metadata = json.loads(raw[8:8 + length])
+                                        if metadata.get("prompt_id") == self.reservation["prompt_id"]:
+                                            extension = ".png" if metadata.get("image_type") == "image/png" else ".jpg"
+                                            image_bytes = raw[8 + length:]
+                                if image_bytes:
+                                    path = self.args.state_root / ("preview-" + token + extension)
+                                    temporary = path.with_suffix(".tmp")
+                                    temporary.write_bytes(image_bytes)
+                                    os.replace(temporary, path)
+                                    self.preview = str(path)
+        try:
+            asyncio.run(read())
+        except Exception as exc:
+            print("Preview/progress connection unavailable: " + str(exc), flush=True)
+        finally:
+            self.events_ready.set()
+
+    def status(self, body):
+        with self.lock:
+            self.verify_token(body)
+            self.check_identity()
+            prompt_id = self.reservation["prompt_id"]
+            if prompt_id is None:
+                return {"state": "reserved"}
+            history = comfy_request(self.url + "/history/" + prompt_id)
+            if prompt_id in history:
+                return {"state": "finished", "history": history[prompt_id], "identity": self.identity(), "nodes": self.progress, "preview": self.preview}
+            queue = comfy_request(self.url + "/queue?deadline_local=1")
+            state = "running" if any(item[1] == prompt_id for item in queue["queue_running"]) else (
+                "pending" if any(item[1] == prompt_id for item in queue["queue_pending"]) else "missing")
+            return {"state": state, "identity": self.identity(), "nodes": self.progress, "preview": self.preview}
+
+    def cancel(self, body):
+        with self.lock:
+            self.verify_token(body)
+            if self.reservation["prompt_id"]:
+                self.check_identity()
+                return comfy_request(self.url + "/api/jobs/" + self.reservation["prompt_id"] + "/cancel", {})
+            self.clear_reservation()
+            if not self.borrowed and not self.args.exclusive_worker:
+                self.stop()  # Release GPU before Deadline releases its task slot.
+            return {"cancelled": True}
+
+    def release(self, body):
+        with self.lock:
+            result = self.status(body)
+            if result["state"] not in {"finished", "missing"}:
+                raise ValueError("Cannot release a reservation while its prompt is active")
+            self.clear_reservation()
+            if not self.borrowed and not self.args.exclusive_worker:
+                self.stop()
+            return {"released": True}
+
+    def stop(self):
+        self.events_stop.set()
+        if self.owned:
+            self.owned.close()
+            self.owned = None
+        if self.log:
+            self.log.close()
+            self.log = None
+        self.session_id = self.endpoint_identity = None
+
+    def idle_tick(self):
+        with self.lock:
+            if self.owned and self.owned.process.poll() is not None:
+                self.stop()
+                self.clear_reservation()
+            if not self.owned and not self.borrowed:
+                return
+            if self.reservation:
+                # A vanished waiter/heartbeat is never permission to kill a
+                # rendering prompt. Reconcile only the exact recorded prompt.
+                token = self.reservation["token"]
+                if self.reservation["prompt_id"] is None and time.time() - self.reservation["reserved_at"] > 120 and self.assignment_active() is False:
+                    self.clear_reservation()
+                    if not self.borrowed and not self.args.exclusive_worker:
+                        self.stop()
+                    return
+                if self.reservation["prompt_id"]:
+                    # Preserve the client-heartbeat timestamp while observing.
+                    last_client = self.reservation["last_client_at"]
+                    result = self.status({"token": token})
+                    self.reservation["last_client_at"] = last_client
+                    if time.monotonic() - last_client > 15:
+                        # Task/sandbox death does not implicitly kill a session.
+                        # Verify the authoritative assignment before interrupting
+                        # only this reservation's prompt. Heartbeat age alone
+                        # cannot authorize terminating active work.
+                        self.reservation.setdefault("last_assignment_check", 0)
+                        if time.monotonic() - self.reservation["last_assignment_check"] > 5:
+                            self.reservation["last_assignment_check"] = time.monotonic()
+                            if self.assignment_active() is False:
+                                if result["state"] not in {"finished", "missing"}:
+                                    self.check_identity()
+                                    comfy_request(self.url + "/api/jobs/" + self.reservation["prompt_id"] + "/cancel", {})
+                                else:
+                                    self.clear_reservation()
+                                    if not self.borrowed and not self.args.exclusive_worker:
+                                        self.stop()
+                return
+            if time.monotonic() - self.last_active < self.args.idle_seconds:
+                return
+            self.draining = True
+            queue = comfy_request(self.url + "/queue?deadline_local=1")
+            if queue["queue_running"] or queue["queue_pending"]:
+                self.draining = False
+                return
+            if not self.borrowed:
+                self.stop()
+            self.draining = False
+
+
+def serve(args):
+    os.environ["DEADLINE_SLAVE_NAME"] = args.worker
+    os.environ["GIT_PYTHON_REFRESH"] = "quiet"
+    owner = SessionOwner(args)
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *_args):
+            pass
+
+        def do_GET(self):
+            with owner.lock:
+                self.respond(200, owner.identity())
+
+        def do_POST(self):
+            try:
+                length = int(self.headers.get("Content-Length", 0))
+                if not 0 < length <= 16 * 1024 ** 2:
+                    raise ValueError("Invalid request size")
+                body = json.loads(self.rfile.read(length))
+                actions = {"/reserve": owner.reserve, "/queue": owner.queue, "/status": owner.status,
+                    "/cancel": owner.cancel, "/release": owner.release}
+                if self.path not in actions:
+                    self.respond(404, {"error": "Unknown session action"})
+                    return
+                self.respond(200, actions[self.path](body))
+            except Exception as exc:
+                self.respond(409, {"error": str(exc)})
+
+        def respond(self, status, data):
+            raw = json.dumps(data).encode()
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(raw)))
+            self.end_headers()
+            self.wfile.write(raw)
+
+    server = ThreadingHTTPServer(("127.0.0.1", args.broker_port), Handler)
+    server.timeout = 1
+    try:
+        while True:
+            server.handle_request()
+            try:
+                owner.idle_tick()
+            except (urllib.error.URLError, TimeoutError) as exc:
+                print(f"Session endpoint unavailable; retaining ownership: {exc}", flush=True)
+    finally:
+        with owner.lock:
+            owner.stop()
+        server.server_close()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--worker", required=True)
+    parser.add_argument("--gpu-uuid", required=True)
+    parser.add_argument("--comfy-root", type=Path, required=True)
+    parser.add_argument("--state-root", type=Path, required=True)
+    parser.add_argument("--comfy-port", type=int, default=8198)
+    parser.add_argument("--broker-port", type=int, default=8201)
+    parser.add_argument("--idle-seconds", type=float, default=60)
+    parser.add_argument("--startup-seconds", type=float, default=120)
+    parser.add_argument("--preview-method", choices=("none", "auto", "latent2rgb", "taesd"), default="auto")
+    parser.add_argument("--exclusive-worker", action="store_true", help="Keep VRAM warm only on Workers reserved for ComfyUI")
+    parser.add_argument("--borrowed-url", default="")
+    parser.add_argument("--borrowed-input-root", type=Path)
+    parser.add_argument("--borrowed-output-root", type=Path)
+    parser.add_argument("--borrowed-temp-root", type=Path)
+    parser.add_argument("--custom-nodes", nargs="*", default=[])
+    parser.add_argument("--extra-args", nargs="*", default=[])
+    serve(parser.parse_args())

native_jobs_adapter.py

@@ -0,0 +1,221 @@
+"""Plugin-owned Jobs API middleware; never inserts farm work in PromptQueue."""
+
+import json
+import uuid
+
+from aiohttp import web
+
+
+STATUSES = {"pending", "in_progress", "completed", "failed", "cancelled"}
+TERMINAL = {"completed", "failed", "cancelled"}
+
+
+def validate_job_id(value):
+    if not isinstance(value, str) or str(uuid.UUID(value)) != value:
+        raise ValueError("job id must be a UUID in canonical lowercase hyphenated form")
+    return value
+
+
+def select_jobs(jobs, query):
+    statuses = None
+    if query.get("status"):
+        statuses = {s.strip().lower() for s in query["status"].split(",") if s.strip()}
+        if statuses - STATUSES:
+            raise ValueError("Invalid status value(s): " + ", ".join(sorted(statuses - STATUSES)))
+    sort_by = query.get("sort_by", "created_at").lower()
+    order = query.get("sort_order", "desc").lower()
+    if sort_by not in {"created_at", "execution_duration"}:
+        raise ValueError("sort_by must be 'created_at' or 'execution_duration'")
+    if order not in {"asc", "desc"}:
+        raise ValueError("sort_order must be 'asc' or 'desc'")
+    try:
+        limit = int(query["limit"]) if "limit" in query else None
+        offset = max(0, int(query.get("offset", 0)))
+    except (ValueError, TypeError):
+        raise ValueError("limit and offset must be integers") from None
+    if limit is not None and limit <= 0:
+        raise ValueError("limit must be a positive integer")
+
+    filtered = [job for job in jobs if (statuses is None or job["status"] in statuses)
+                and (not query.get("workflow_id") or job.get("workflow_id") == query["workflow_id"])]
+    if sort_by == "execution_duration":
+        def sort_key(job):
+            start, end = job.get("execution_start_time"), job.get("execution_end_time")
+            return end - start if start and end else 0
+    else:
+        def sort_key(job):
+            return job.get("create_time") or 0
+    filtered.sort(key=sort_key, reverse=order == "desc")
+    total = len(filtered)
+    page = filtered[offset:] if limit is None else filtered[offset:offset + limit]
+    return {"jobs": page, "pagination": {"offset": offset, "limit": limit,
+            "total": total, "has_more": offset + len(page) < total}}
+
+
+def response_data(response):
+    return json.loads(response.body)
+
+
+class NativeJobsAdapter:
+    """Provider methods use cached records and must not block the aiohttp loop."""
+
+    def __init__(self, provider):
+        self.provider = provider
+
+    @web.middleware
+    async def middleware(self, request, handler):
+        path = request.path
+        if path.startswith("/api/"):
+            path = path[4:]
+
+        if request.method == "GET" and path == "/view" and request.query.get("subfolder", "").startswith("deadline-native/"):
+            if hasattr(self.provider, "view"):
+                return await self.provider.view(request)
+            return web.Response(status=404)
+
+        if request.method == "POST" and path == "/prompt":
+            body = await request.json()
+            workflow = body.get("extra_data", {}).get("extra_pnginfo", {}).get("workflow", {}) if isinstance(body, dict) else {}
+            if isinstance(workflow, dict) and "__deadline_native_queue" in workflow:
+                return web.json_response({"error": {"type": "deadline_routing_error",
+                    "message": "A Deadline snapshot reached the local prompt endpoint. Check extension compatibility.",
+                    "details": "The prompt was not queued locally."}, "node_errors": {}}, status=409)
+
+        if request.method == "GET" and path == "/jobs":
+            try:
+                select_jobs([], request.query)
+            except ValueError as exc:
+                return web.json_response({"error": str(exc)}, status=400)
+            # Fetch the complete local set. Pagination applies once, after merging.
+            local = await handler(request.clone(rel_url=request.rel_url.with_query({})))
+            if local.status != 200:
+                return local
+            farm = await self.provider.list_jobs()
+            jobs = {job["id"]: job for job in response_data(local)["jobs"]}
+            for job in farm:
+                if job["id"] in jobs:
+                    return web.json_response({"error": "Farm/local run ID collision"}, status=409)
+                jobs[job["id"]] = job
+            return web.json_response(select_jobs(list(jobs.values()), request.query))
+
+        parts = path.strip("/").split("/")
+        if len(parts) in {2, 3} and parts[0] == "jobs" and parts[1] != "cancel":
+            job = await self.provider.get_job(parts[1])
+            if job is not None:
+                if request.method == "GET" and len(parts) == 2:
+                    return web.json_response(job)
+                if request.method == "GET" and parts[2:] == ["assets"]:
+                    try:
+                        limit = int(request.query.get("limit", 500))
+                        offset = max(0, int(request.query.get("offset", 0)))
+                        if not 0 < limit <= 500:
+                            raise ValueError("assets limit must be between 1 and 500")
+                    except ValueError as exc:
+                        return web.json_response({"error": str(exc)}, status=400)
+                    assets = await self.provider.get_assets(parts[1])
+                    page = assets[offset:offset + limit]
+                    return web.json_response({"job_id": parts[1], "assets": page,
+                        "pagination": {"offset": offset, "limit": limit,
+                            "total": len(assets), "has_more": offset + len(page) < len(assets)}})
+                if request.method == "POST" and parts[2:] == ["cancel"]:
+                    return web.json_response({"cancelled": await self.provider.cancel(parts[1])})
+
+        if request.method == "POST" and path == "/jobs/cancel":
+            try:
+                body = await request.json()
+                ids = body.get("job_ids") if isinstance(body, dict) else None
+                if not isinstance(ids, list):
+                    raise ValueError("job_ids must be a list")
+                for job_id in ids:
+                    validate_job_id(job_id)
+            except (ValueError, AttributeError) as exc:
+                return web.json_response({"error": str(exc)}, status=400)
+            # The stock handler treats unknown farm UUIDs as no-ops; it retains
+            # responsibility for local IDs and its atomic targeted interruption.
+            local = await handler(request)
+            if local.status != 200:
+                return local
+            cancelled = response_data(local)["cancelled"]
+            for job_id in dict.fromkeys(ids):
+                if await self.provider.get_job(job_id) is not None:
+                    cancelled = await self.provider.cancel(job_id) or cancelled
+            return web.json_response({"cancelled": cancelled})
+
+        if request.method == "POST" and path == "/interrupt":
+            try:
+                body = await request.json() if request.can_read_body else {}
+            except ValueError:
+                return await handler(request)
+            job_id = body.get("prompt_id") if isinstance(body, dict) else None
+            if isinstance(job_id, str) and await self.provider.get_job(job_id) is not None:
+                await self.provider.cancel(job_id)
+                return web.Response(status=200)
+
+        if path == "/queue":
+            if request.method == "GET":
+                local = await handler(request)
+                if request.query.get("deadline_local") == "1":
+                    return local  # Session drain observes actual execution only.
+                if local.status != 200:
+                    return local
+                data = response_data(local)
+                for job in await self.provider.list_jobs():
+                    if job["status"] in {"pending", "in_progress"}:
+                        history = await self.provider.get_history(job["id"])
+                        key = "queue_running" if job["status"] == "in_progress" else "queue_pending"
+                        data[key].append(history["prompt"])
+                return web.json_response(data)
+            if request.method == "POST":
+                body = await request.json()
+                local = await handler(request)
+                if local.status != 200:
+                    return local
+                ids = body.get("delete", [])
+                if body.get("clear"):
+                    ids = [j["id"] for j in await self.provider.list_jobs() if j["status"] == "pending"]
+                for job_id in ids:
+                    if await self.provider.get_job(job_id) is not None:
+                        await self.provider.cancel(job_id)
+                return local
+
+        if request.method == "GET" and parts[0] == "history":
+            if len(parts) == 2:
+                job = await self.provider.get_job(parts[1])
+                if job is not None and job["status"] in TERMINAL:
+                    return web.json_response({parts[1]: await self.provider.get_history(parts[1])})
+            elif len(parts) == 1:
+                local = await handler(request.clone(rel_url=request.rel_url.with_query({})))
+                if local.status != 200:
+                    return local
+                records = response_data(local)
+                for job in await self.provider.list_jobs():
+                    if job["status"] in TERMINAL:
+                        records[job["id"]] = await self.provider.get_history(job["id"])
+                try:
+                    limit = int(request.query["max_items"]) if "max_items" in request.query else None
+                    offset = int(request.query.get("offset", -1))
+                    if limit is not None and limit < 0:
+                        raise ValueError("max_items must be non-negative")
+                except ValueError as exc:
+                    return web.json_response({"error": str(exc)}, status=400)
+                items = sorted(records.items(), key=lambda item: item[1]["prompt"][3].get("create_time", 0))
+                if offset < 0:
+                    offset = max(0, len(items) - limit) if limit is not None else 0
+                items = items[offset:] if limit is None else items[offset:offset + limit]
+                return web.json_response(dict(items))
+
+        if request.method == "POST" and path == "/history":
+            body = await request.json()
+            local = await handler(request)
+            if local.status != 200:
+                return local
+            ids = body.get("delete", [])
+            if body.get("clear"):
+                ids = [j["id"] for j in await self.provider.list_jobs() if j["status"] in TERMINAL]
+            for job_id in ids:
+                job = await self.provider.get_job(job_id)
+                if job is not None and job["status"] in TERMINAL:
+                    await self.provider.hide(job_id)
+            return local
+
+        return await handler(request)

plugins/ComfyUI/ComfyUI.param

@@ -39,3 +39,19 @@ Category=Configuration
 Index=2
 Default={}
 Description=Optional JSON keyed by worker name. Example: {"M28":{"ComfyUIApiUrl":"http://127.0.0.1:8188","LaunchGpuUuid":"GPU-..."}}. A verified endpoint is reused; fallback launches only at that port and resolves the UUID.
+
+[NativeWorkerSessions]
+Type=multilinestring
+Label=Native Queue Session Owners
+Category=Configuration
+Index=3
+Default={}
+Description=JSON keyed by Deadline Worker. Each entry contains a loopback broker and stable gpu_uuid. Example: {"M28":{"broker":"http://127.0.0.1:8202","gpu_uuid":"GPU-..."}}. Install the session owner independently with scripts/install_native_session.ps1. Warm mode requires exclusive ComfyUI Workers.
+
+[NativeQueueRoot]
+Type=filename
+Label=Native Queue Shared Storage
+Category=Configuration
+Index=4
+Default=
+Description=Off-host UNC folder for durable native queue runs, inputs and results. Submission and Worker accounts need read/write access. Normal tasks use the configured ComfyUI installation and Deadline GPU affinity; session owners are optional.

plugins/ComfyUI/ComfyUI.py

@@ -278,6 +278,9 @@ class ComfyUI(DeadlinePlugin):
 
     def Cleanup(self):
         """Clean up plugin resources"""
+        if getattr(self, "native_spec", None):
+            import native_queue_task
+            native_queue_task.cleanup(self)
         self.thread_running = False
         
         # Clean up callbacks
@@ -704,6 +707,17 @@ class ComfyUI(DeadlinePlugin):
 
     def PreRenderTasks(self):
         """Setup tasks before rendering"""
+        if self.GetPluginInfoEntryWithDefault("NativeQueueProtocol", ""):
+            if self.GetPluginInfoEntryWithDefault("NativeQueueProtocol", "") != "1":
+                raise ComfyUIError("Unsupported native queue protocol")
+            # Deadline copies helpers but does not add the sandbox directory
+            # to Python's import path. Scope this to the explicit native mode.
+            helper_directory = os.path.dirname(os.path.abspath(__file__))
+            if helper_directory not in sys.path:
+                sys.path.insert(0, helper_directory)
+            import native_queue_task
+            native_queue_task.prepare(self)
+            return
         self.LogInfo("ComfyUI PreRenderTasks started.")
         
         try:
@@ -750,6 +764,14 @@ class ComfyUI(DeadlinePlugin):
 
     def PostRenderTasks(self):
         """Cleanup tasks after rendering"""
+        if getattr(self, "native_spec", None):
+            from farm_protocol import RunStore
+            import native_queue_task
+            native_queue_task.cleanup(self)
+            state = RunStore(self.native_spec["root"]).control(self.native_spec["run_id"])
+            if not state.get("result") and not state.get("cancel_requested"):
+                self.FailRender("Native task ended without a committed result")
+            return
         self.LogInfo("ComfyUI PostRenderTasks started.")
 
         if not self.task_completed:
@@ -820,6 +842,9 @@ class ComfyUI(DeadlinePlugin):
 
     def RenderArgument(self):
         """Build command line arguments for ComfyUI"""
+        if getattr(self, "native_spec", None):
+            script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "native_queue_task.py")
+            return subprocess.list2cmdline(["-s", script, self.native_spec_path])
         comfyui_path = self._resolve_comfyui_install_path()
         if not comfyui_path:
             return ""

plugins/ComfyUI/native_queue_assignment.py

@@ -0,0 +1,10 @@
+"""Deadline CLI query with exact .NET task-start ticks for result fencing."""
+import json
+
+
+def __main__(*args):
+    from Deadline.Scripting import RepositoryUtils
+    job = RepositoryUtils.GetJob(args[0], True)
+    task = next(t for t in RepositoryUtils.GetJobTasks(job, True).Tasks if str(t.TaskId) == "0")
+    print("COMFY_NATIVE_ASSIGNMENT=" + json.dumps({"worker": str(task.TaskSlaveName),
+        "status": str(task.TaskStatus), "start": int(task.TaskStartTime.ToUniversalTime().Ticks)}))

plugins/ComfyUI/native_queue_task.py

@@ -0,0 +1,383 @@
+"""Explicit native-queue mode for the existing ComfyUI Deadline plugin.
+
+The sandbox prepares with the existing preflight. A small supervised client
+tracks only its reservation/prompt and commits validated files before success.
+"""
+import argparse
+import copy
+import hashlib
+import json
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import uuid
+from pathlib import Path
+
+# Windows portable Python uses an isolated ._pth and omits the script directory.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from farm_protocol import RunStore, SCHEMA, atomic_json, canonical_id, digest, read_json, safe_path
+
+
+def free_port():
+    with socket.socket() as listener:
+        listener.bind(("127.0.0.1", 0))
+        return listener.getsockname()[1]
+
+
+def start_task_owner(plugin, startup_seconds=120):
+    """A cold render owns its supervisor and GPU child for this task only."""
+    from farm_session_owner import OwnedProcess
+    plugin._set_deadline_environment_variables()
+    portable = Path(plugin._resolve_comfyui_install_path())
+    gpu = plugin._get_gpu_from_worker_affinity()
+    if gpu is None:
+        gpu = plugin._get_default_cuda_device()
+    if gpu is None:
+        raise ValueError("Configure a Deadline GPU affinity or default CUDA device")
+    output = subprocess.check_output(["nvidia-smi", "--query-gpu=index,uuid", "--format=csv,noheader"], text=True, timeout=10)
+    devices = dict([value.strip() for value in line.split(",", 1)] for line in output.splitlines() if line.strip())
+    if str(gpu) not in devices:
+        raise ValueError("Deadline assigned a GPU which is not present on this Worker")
+    broker_port, comfy_port = free_port(), free_port()
+    while comfy_port == broker_port:
+        comfy_port = free_port()
+    state = Path(plugin.temp_dir) / "session"
+    state.mkdir(parents=True, exist_ok=True)
+    plugin.native_owner_log = (state / "owner.log").open("ab", buffering=0)
+    command = [str(portable / "python_embeded" / "python.exe"), "-s",
+        str(Path(__file__).with_name("farm_session_owner.py")),
+        "--worker", plugin.GetSlaveName(), "--gpu-uuid", devices[str(gpu)],
+        "--comfy-root", str(portable / "ComfyUI"), "--state-root", str(state),
+        "--broker-port", str(broker_port), "--comfy-port", str(comfy_port),
+        "--startup-seconds", str(startup_seconds)]
+    plugin.native_owner = OwnedProcess(command, str(portable), plugin.native_owner_log)
+    broker = "http://127.0.0.1:" + str(broker_port)
+    until = time.monotonic() + 15
+    while time.monotonic() < until:
+        if plugin.native_owner.process.poll() is not None:
+            raise RuntimeError("Task session owner exited; inspect " + str(state / "owner.log"))
+        try:
+            identity = request(broker)
+        except (urllib.error.URLError, TimeoutError):
+            time.sleep(.1)
+            continue
+        if identity.get("owner_pid") != plugin.native_owner.process.pid:
+            raise RuntimeError("Task session owner port belongs to another process")
+        return {"broker": broker, "gpu_uuid": devices[str(gpu)], "startup_seconds":startup_seconds}
+    raise TimeoutError("Task session owner did not start")
+
+
+def request(url, body=None, timeout=None):
+    try:
+        with urllib.request.urlopen(urllib.request.Request(url, data=json.dumps(body).encode() if body is not None else None,
+                headers={"Content-Type": "application/json"}), timeout=timeout or (150 if url.endswith("/reserve") else 10)) as response:
+            raw = response.read()
+            return json.loads(raw) if raw else {}
+    except urllib.error.HTTPError as exc:
+        raise RuntimeError(exc.read().decode(errors="replace")) from exc
+
+
+def assignment(plugin):
+    # Deadline's authoritative assigned task and start time fence old waiters.
+    from Deadline.Scripting import RepositoryUtils
+    tasks = RepositoryUtils.GetJobTasks(plugin.GetJob(), True)
+    task = next(t for t in tasks.Tasks if str(t.TaskId) == "0")
+    if str(task.TaskSlaveName) != plugin.GetSlaveName() or str(task.TaskStatus).lower() != "rendering":
+        raise ValueError("Deadline task is not currently assigned to this Worker")
+    start = task.TaskStartTime.ToUniversalTime().Ticks
+    from System.Globalization import CultureInfo
+    return {"worker": plugin.GetSlaveName(), "start": int(start),
+        "display_start": task.TaskStartTime.ToString("MMM dd/yy  HH:mm:ss", CultureInfo.InvariantCulture)}
+
+
+def prepare(plugin):
+    if getattr(plugin, "native_owner", None):
+        cleanup(plugin)
+    run_id = canonical_id(plugin.GetPluginInfoEntry("NativeRunId"))
+    root = plugin.GetPluginInfoEntry("NativeRoot")
+    if not root.startswith("\\\\"):
+        raise ValueError("Native payload storage must be an off-host UNC root")
+    store = RunStore(root)
+    store.check_root()
+    submission = read_json(store.run(run_id) / "submission.json")
+    if submission["schema"] != SCHEMA or digest(submission["payload"]) != submission["snapshot_hash"]:
+        raise ValueError("Unsupported or modified native snapshot")
+    mapping = json.loads(plugin.GetConfigEntryWithDefault("NativeWorkerSessions", "{}"))
+    config = next((v for k, v in mapping.items() if k.lower() == plugin.GetSlaveName().lower()), None)
+    if config is None:
+        from Deadline.Scripting import RepositoryUtils
+        settings = RepositoryUtils.GetSlaveSettings(plugin.GetSlaveName(), True)
+        registration = settings.GetSlaveExtraInfoKeyValueWithDefault("ComfyNativeSession", "")
+        config = json.loads(registration) if registration else None
+    attempt_id = str(uuid.uuid4())
+    job_id = str(plugin.GetJob().JobId)
+    state = store.begin_attempt(run_id, attempt_id, job_id, plugin.GetSlaveName(), assignment(plugin),
+        execution_mode="task-owned" if config is None else "persistent")
+    spec = {"root": root, "run_id": run_id, "attempt_id": attempt_id, "job_id": job_id, "skip": state is None}
+    plugin.native_spec = spec
+    plugin.temp_dir = plugin.CreateTempDirectory("comfyui_native_task")
+    if state is not None:
+        spec["generation"] = state["generation"]
+        spec["assignment"] = state["assignment"]
+        try:
+            if config is None:
+                config = start_task_owner(plugin, submission["settings"]["task_timeout_minutes"] * 60)
+            if not isinstance(config, dict):
+                raise ValueError("Invalid NativeWorkerSessions entry for " + plugin.GetSlaveName())
+            broker = config.get("broker", "http://127.0.0.1:8202").rstrip("/")
+            url = urllib.parse.urlparse(broker)
+            if url.scheme != "http" or url.hostname not in {"127.0.0.1", "localhost"} or url.username or url.path:
+                raise ValueError("Session broker must be a worker-local loopback HTTP endpoint")
+            spec["broker"] = broker
+            identity = request(broker + "/reserve", {"worker": plugin.GetSlaveName(), "gpu_uuid": config["gpu_uuid"],
+                "run_id": run_id, "attempt_id": attempt_id, "job_id": job_id}, timeout=config.get("startup_seconds",120) + 30)
+            spec.update(token=identity["reservation"]["token"], identity=identity)
+            plugin.native_spec = spec  # Cleanup can cancel even if preflight raises.
+            if config.get("comfyui_root"):
+                configured_root = Path(config["comfyui_root"])
+                if not (configured_root / "main.py").is_file() or not (configured_root.parent / "python_embeded" / "python.exe").is_file():
+                    raise ValueError("The native session mapping's portable ComfyUI installation is missing")
+                plugin.comfyui_install_path = str(configured_root.parent)
+            portable = Path(plugin._resolve_comfyui_install_path())
+            if Path(identity["comfyui_root"]).resolve() != (portable / "ComfyUI").resolve():
+                raise ValueError("Session owner installation differs from Deadline ComfyUIPath")
+            plugin.configured_launch_gpu_uuid = config["gpu_uuid"]
+            plugin._get_policy_launch_gpu()  # UUID present locally and inside Deadline GPU affinity.
+            payload = copy.deepcopy(submission["payload"])
+            stage_inputs(store, run_id, attempt_id, payload, identity)
+            namespace = "deadline-native/" + run_id + "/" + attempt_id
+            for node in payload["prompt"].values():
+                for name in ("filename_prefix", "output_filename_prefix"):
+                    if name in node["inputs"]:
+                        original = node["inputs"][name]
+                        if not isinstance(original, str):
+                            raise ValueError("Dynamic output filename prefixes are unsupported in native queue mode")
+                        node["inputs"][name] = namespace + "/" + Path(original.replace("\\", "/")).name
+            plugin.use_existing_comfyui = True
+            plugin.reuse_gui_input_root = identity["input_root"]
+            plugin.object_info = request(identity["url"] + "/object_info")
+            workflow = payload["extra_data"]["extra_pnginfo"]["workflow"]
+            payload["prompt"], expected = plugin._preflight_worker_payload(payload["prompt"], workflow)
+            spec.update(payload=payload, expected_outputs=expected, timeout_seconds=submission["settings"]["task_timeout_minutes"] * 60)
+            stats = request(identity["url"] + "/system_stats")
+            store.update_attempt(run_id, attempt_id, state["generation"], session_id=identity["session_id"],
+                session_mode=identity["mode"], worker_pid=identity["pid"], environment=stats.get("system", {}))
+        except Exception as exc:
+            log = Path(plugin.temp_dir) / "session" / "comfy.log"
+            if log.is_file():
+                plugin.LogWarning("ComfyUI startup/preflight log:\n" + "\n".join(log.read_text(encoding="utf-8", errors="replace").splitlines()[-40:]))
+            store.update_attempt(run_id, attempt_id, state["generation"], status="failed",
+                error={"exception_type": type(exc).__name__, "exception_message": str(exc)})
+            cleanup(plugin)
+            raise
+    plugin.native_spec = spec
+    path = Path(plugin.temp_dir) / "task.json"
+    atomic_json(path, spec)  # Reservation token is worker-local, never in shared records.
+    plugin.native_spec_path = str(path)
+    return spec
+
+
+def stage_inputs(store, run_id, attempt_id, payload, identity):
+    manifest = read_json(store.run(run_id) / "input_manifest.json")
+    mapping = {}
+    for asset in manifest["assets"]:
+        source = safe_path(store.run(run_id) / "inputs", asset["staged_relative_path"])
+        with source.open("rb") as handle:
+            checksum = hashlib.sha256(handle.read()).hexdigest()
+        if checksum != asset["sha256"]:
+            raise ValueError("Staged input changed: " + asset["staged_relative_path"])
+        relative = "deadline-native/" + run_id + "/" + attempt_id + "/" + asset["staged_relative_path"]
+        target = safe_path(identity["input_root"], relative)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, target)
+        mapping[asset["staged_relative_path"]] = relative
+        mapping[asset["destination"]] = str(target)
+    for node in payload["prompt"].values():
+        for name, value in node["inputs"].items():
+            if isinstance(value, str) and value in mapping:
+                node["inputs"][name] = mapping[value]
+
+
+def cleanup_reservation(plugin):
+    spec = getattr(plugin, "native_spec", None)
+    if not spec or not spec.get("token"):
+        return
+    try:
+        status = request(spec["broker"] + "/status", {"token": spec["token"]})
+        if status["state"] in {"finished", "missing"}:
+            state = RunStore(spec["root"]).control(spec["run_id"])
+            if state.get("cancel_requested"):
+                RunStore(spec["root"]).update_attempt(spec["run_id"], spec["attempt_id"], spec["generation"],
+                    status="cancelled", finished_at=int(time.time() * 1000))
+            request(spec["broker"] + "/release", {"token": spec["token"]})
+        else:
+            request(spec["broker"] + "/cancel", {"token": spec["token"]})
+            for _ in range(20):
+                status = request(spec["broker"] + "/status", {"token": spec["token"]})
+                if status["state"] in {"finished", "missing"}:
+                    state = RunStore(spec["root"]).control(spec["run_id"])
+                    if state.get("cancel_requested"):
+                        RunStore(spec["root"]).update_attempt(spec["run_id"], spec["attempt_id"], spec["generation"],
+                            status="cancelled", finished_at=int(time.time() * 1000))
+                    request(spec["broker"] + "/release", {"token": spec["token"]})
+                    break
+                time.sleep(.5)
+    except Exception as exc:
+        if "Reservation token does not own this session" in str(exc):
+            return
+        plugin.LogWarning("Native reservation cleanup: " + str(exc))
+
+
+def cleanup(plugin):
+    try:
+        cleanup_reservation(plugin)
+    finally:
+        owner = getattr(plugin, "native_owner", None)
+        if owner is not None:
+            plugin.native_owner = None
+            owner.close()
+        log = getattr(plugin, "native_owner_log", None)
+        if log is not None:
+            plugin.native_owner_log = None
+            log.close()
+
+
+def publish_outputs(store, spec, history):
+    outputs = copy.deepcopy(history["outputs"])
+    files, seen_nodes = [], set()
+    for node_id, groups in outputs.items():
+        for group, items in groups.items():
+            if not isinstance(items, list):
+                continue
+            for index, item in enumerate(items):
+                if not isinstance(item, dict) or "filename" not in item:
+                    continue
+                kind = item.get("type", "output")
+                if kind not in {"output", "temp"}:
+                    raise ValueError("Unsupported output file type: " + kind)
+                source = safe_path(spec["identity"][kind + "_root"], item.get("subfolder", "") + "/" + item["filename"] if item.get("subfolder") else item["filename"])
+                if not source.is_file() or source.stat().st_size <= 0:
+                    raise ValueError("Missing ComfyUI output: " + str(source))
+                relative = "attempts/" + spec["attempt_id"] + "/outputs/" + str(node_id) + "/" + group + "/" + str(index) + "-" + Path(item["filename"]).name
+                destination = safe_path(store.run(spec["run_id"]), relative)
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(source, destination)
+                with destination.open("rb") as handle:
+                    checksum = hashlib.sha256(handle.read()).hexdigest()
+                files.append({"path": relative, "size": destination.stat().st_size, "sha256": checksum,
+                    "node_id": str(node_id), "output_key": group, "output_index": index})
+                seen_nodes.add(str(node_id))
+    missing = set(spec["expected_outputs"]) - seen_nodes
+    if missing:
+        raise ValueError("Output nodes produced no validated files: " + ", ".join(sorted(missing)))
+    return {"schema": SCHEMA, "run_id": spec["run_id"], "attempt_id": spec["attempt_id"],
+        "generation": spec["generation"], "files": files, "outputs": outputs,
+        "execution_status": history["status"], "session_id": spec["identity"]["session_id"]}
+
+
+def run(spec):
+    store = RunStore(spec["root"])
+    run_id = spec["run_id"]
+    if spec["skip"]:
+        if store.control(run_id).get("cancel_requested"):
+            return
+        if not store.result(run_id):
+            raise RuntimeError("Skipped assignment without a valid accepted result")
+        print("Progress: 100%", flush=True)
+        return
+    attempt, generation, token = spec["attempt_id"], spec["generation"], spec["token"]
+    broker = spec["broker"]
+    try:
+        control = store.control(run_id)
+        if control.get("cancel_requested") or not store.owns(control, attempt, generation):
+            request(broker + "/cancel", {"token": token})
+            return
+        queued = request(broker + "/queue", {"token": token, "payload": spec["payload"]})
+        store.update_attempt(run_id, attempt, generation, worker_prompt_id=queued["prompt_id"])
+        until, sequence = time.monotonic() + spec["timeout_seconds"], 0
+        while time.monotonic() < until:
+            control = store.control(run_id)
+            if not store.owns(control, attempt, generation):
+                request(broker + "/cancel", {"token": token})
+                raise RuntimeError("Attempt was superseded")
+            status = request(broker + "/status", {"token": token})
+            if status["state"] == "missing":
+                raise RuntimeError("The reserved Worker prompt is absent from queue and history")
+            if control.get("cancel_requested") and status["state"] != "finished":
+                request(broker + "/cancel", {"token": token})
+            sequence += 1
+            event = {"run_id": run_id, "generation": generation, "sequence": sequence,
+                "status": "in_progress", "nodes": status.get("nodes", {})}
+            if status.get("preview"):
+                source = Path(status["preview"])
+                relative = "attempts/" + attempt + "/preview" + source.suffix
+                destination = safe_path(store.run(run_id), relative)
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                temporary = destination.with_suffix(".tmp")
+                shutil.copy2(source, temporary)
+                os.replace(temporary, destination)
+                event["preview_url"] = "/deadline/farm/" + run_id + "/preview?generation=" + str(generation) + "&sequence=" + str(sequence)
+                event["preview_file"] = relative
+            atomic_json(store.run(run_id) / "attempts" / attempt / "progress.json", event)
+            nodes = event["nodes"]
+            if nodes:
+                fraction = sum(min(n.get("value", 0) / max(n.get("max", 1), 1), 1) for n in nodes.values()) / len(nodes)
+                print("Progress: " + str(round(fraction * 99, 1)) + "%", flush=True)
+            if status["state"] == "finished":
+                if control.get("cancel_requested"):
+                    store.update_attempt(run_id, attempt, generation, status="cancelled", finished_at=int(time.time() * 1000))
+                    request(broker + "/release", {"token": token})
+                    return
+                history = status["history"]
+                if history["status"]["status_str"] != "success":
+                    errors = [m[1] for m in history["status"]["messages"] if m[0] == "execution_error"]
+                    error = errors[-1] if errors else {"exception_type": "ComfyUIError", "exception_message": str(history["status"])}
+                    store.update_attempt(run_id, attempt, generation, status="failed", error=error)
+                    raise RuntimeError(error.get("exception_message", "ComfyUI prompt failed"))
+                manifest = publish_outputs(store, spec, history)
+                if not store.commit(run_id, attempt, generation, manifest, lambda: assignment_is_current(spec)):
+                    if store.control(run_id).get("cancel_requested"):
+                        store.update_attempt(run_id, attempt, generation, status="cancelled", finished_at=int(time.time() * 1000))
+                    else:
+                        raise RuntimeError("Result publication was fenced out")
+                request(broker + "/release", {"token": token})
+                print("Progress: 100%", flush=True)
+                return
+            time.sleep(1)
+        request(broker + "/cancel", {"token": token})
+        raise TimeoutError("Native task timeout; its own prompt was cancelled")
+    except Exception as exc:
+        store.update_attempt(run_id, attempt, generation, status="failed",
+            error={"exception_type": type(exc).__name__, "exception_message": str(exc)})
+        try:
+            request(broker + "/cancel", {"token": token})
+        except Exception:
+            pass
+        raise
+
+
+def assignment_is_current(spec):
+    command = Path(os.environ.get("DEADLINE_PATH", r"C:\Program Files\Thinkbox\Deadline10\bin")) / "deadlinecommand.exe"
+    helper = Path(__file__).with_name("native_queue_assignment.py")
+    checked = subprocess.run([str(command), "-ExecuteScript", str(helper), spec["job_id"]], capture_output=True,
+        text=True, timeout=30, creationflags=subprocess.CREATE_NO_WINDOW)
+    if checked.returncode or any(line.startswith("Error:") for line in checked.stdout.splitlines()):
+        raise RuntimeError("Could not verify the Deadline assignment before result publication")
+    records = [json.loads(line.split("=", 1)[1]) for line in checked.stdout.splitlines() if line.startswith("COMFY_NATIVE_ASSIGNMENT=")]
+    if len(records) != 1:
+        raise RuntimeError("Deadline did not return an exact task assignment")
+    fields = records[0]
+    assignment = spec["assignment"]
+    return fields.get("status", "").lower() == "rendering" and fields.get("worker") == assignment["worker"] and fields.get("start") == assignment["start"]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("spec", type=Path)
+    run(read_json(parser.parse_args().spec))

pyproject.toml

@@ -33,7 +33,7 @@ Issues = "https://github.com/doubletwisted/ComfyUI-Deadline-Plugin/issues"
 
 [tool.setuptools]
 # For flat structure with .py files in root
-py-modules = ["deadline_submit", "__init__"]
+py-modules = ["deadline_submit", "native_jobs_adapter", "farm_protocol", "farm_backend", "farm_readiness", "farm_session_owner", "__init__"]
 
 [comfyui]
 PublisherId = "doubletwisted"

scripts/deadline_farm_snapshot.py

@@ -0,0 +1,72 @@
+"""Read-only repository snapshot, executed by Deadline's own Python runtime."""
+import json
+import os
+import re
+from urllib.parse import urlsplit
+from Deadline.Scripting import RepositoryUtils
+
+
+def session_mapping(mapping):
+    if not isinstance(mapping, dict):
+        return None, "Session mapping must be a JSON object" if mapping is not None else None
+    if not re.fullmatch(r"GPU-[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}", str(mapping.get("gpu_uuid", ""))):
+        return None, "Session mapping needs a valid NVIDIA GPU UUID"
+    try:
+        broker = urlsplit(mapping.get("broker", "http://127.0.0.1:8202"))
+        if broker.scheme != "http" or broker.hostname not in {"127.0.0.1", "localhost"} or broker.username or broker.password or broker.path not in {"", "/"} or broker.query or broker.fragment or not 1024 <= (broker.port or 80) <= 65535:
+            raise ValueError("Session broker must be a Worker-local loopback HTTP endpoint")
+    except (ValueError, TypeError) as exc:
+        return None, str(exc)
+    return mapping, None
+
+
+def __main__(*args):
+    repository = str(RepositoryUtils.GetRepositoryPath("", False))
+    config_error = None
+    try:
+        plugin_config = RepositoryUtils.GetPluginConfig("ComfyUI")
+        sessions = json.loads(plugin_config.GetConfigEntryWithDefault("NativeWorkerSessions", "{}"))
+        if not isinstance(sessions, dict):
+            raise ValueError("NativeWorkerSessions must be a JSON object")
+        sessions = {str(name).lower(): mapping for name, mapping in sessions.items()}
+    except Exception as exc:
+        sessions, config_error = {}, str(exc)
+    settings = {str(item.SlaveName): item for item in RepositoryUtils.GetSlaveSettingsList(True)}
+    workers = []
+    for info in RepositoryUtils.GetSlaveInfos(True):
+        name = str(info.SlaveName)
+        item = settings.get(name)
+        mapping = sessions.get(name.lower())
+        error = None
+        if mapping is None and item:
+            try:
+                extra = str(item.GetSlaveExtraInfoKeyValueWithDefault("ComfyNativeSession", ""))
+                mapping = json.loads(extra) if extra else None
+            except Exception as exc:
+                error = str(exc)
+        mapping, mapping_error = session_mapping(mapping)
+        error = error or mapping_error
+        mapped = mapping is not None
+        workers.append({"name": name, "host": str(info.HostName), "status": str(info.SlaveStatus),
+            "enabled": bool(item.SlaveEnabled) if item else False,
+            "scheduling_mode": str(item.SlaveSchedulingMode) if item else "Unknown",
+            "user_job_names": [str(user) for user in item.SlaveUserJobsModeNames] if item else [],
+            "machine_user": str(info.MachineUserName),
+            "pools": [str(p) for p in item.SlavePools] if item else [],
+            "groups": [str(g) for g in item.SlaveGroups] if item else [],
+            "gpu_model": str(mapping.get("gpu_name", "")) if mapped else "",
+            "display_adapter": str(info.MachineVideoCard),
+            "gpu_affinity": [int(g) for g in item.SlaveGpuAffinity] if item and item.SlaveOverrideGpuAffinity else [],
+            "session_configured": mapped, "session_error": error,
+            "gpu_uuid": str(mapping.get("gpu_uuid", "")) if mapped else "",
+            "broker": str(mapping.get("broker", "http://127.0.0.1:8202")) if mapped else "",
+            "managed_local": bool(item.GetSlaveExtraInfoKeyValueWithDefault("ComfyNativeManaged", "")) if item else False})
+    plugin_parent = str(args[0]) if args and str(args[0]).strip() else os.path.join(repository, "custom", "plugins")
+    plugin = os.path.join(plugin_parent, "ComfyUI")
+    available = all(os.path.isfile(os.path.join(plugin, name)) for name in ("ComfyUI.py", "ComfyUI.param", "native_queue_task.py", "native_queue_assignment.py", "farm_protocol.py", "farm_session_owner.py"))
+    print("COMFY_NATIVE_FARM=" + json.dumps({"repository": repository,
+        "pools": sorted(str(p) for p in RepositoryUtils.GetPoolNames()),
+        "groups": sorted(str(g) for g in RepositoryUtils.GetGroupNames()),
+        "workers": sorted(workers, key=lambda w: w["name"].lower()),
+        "native_root": RepositoryUtils.GetPluginConfig("ComfyUI").GetConfigEntryWithDefault("NativeQueueRoot", ""),
+        "plugin_available": available, "plugin_directory": plugin_parent, "configuration_error": config_error}))

scripts/deadline_job_status.py

@@ -0,0 +1,15 @@
+"""Confirm absent Jobs against fresh repository records, including deleted Jobs."""
+import json
+from Deadline.Scripting import RepositoryUtils
+
+
+def __main__(*job_ids):
+    records = {}
+    for job_id in job_ids:
+        job = RepositoryUtils.GetJob(job_id, True)
+        if job is not None:
+            records[job_id] = {"Status": str(job.JobStatus)}
+        else:
+            deleted = RepositoryUtils.GetDeletedJob(job_id)
+            records[job_id] = {"Status": "Deleted" if deleted is not None else "Missing"}
+    print("COMFY_NATIVE_JOBS=" + json.dumps(records))

scripts/deploy_deadline_plugin.ps1

@@ -177,5 +177,15 @@ if (!$NoBackup -and (Test-Path -LiteralPath $destinationPluginDirectory)) {
 
 Copy-PluginDirectory -SourceDirectory $sourcePluginDirectory -DestinationDirectory $destinationPluginDirectory -ShouldClean ([bool]$Clean)
 
+# The same protocol implementation must be importable inside Deadline's copied
+# sandbox. Do not depend on a submitting workstation's custom-node sys.path.
+if ($PluginName -eq "ComfyUI") {
+    foreach ($helper in @("farm_protocol.py", "farm_session_owner.py")) {
+        if ($PSCmdlet.ShouldProcess($destinationPluginDirectory, "Copy native queue helper $helper")) {
+            Copy-Item -LiteralPath (Join-Path $projectRoot $helper) -Destination (Join-Path $destinationPluginDirectory $helper) -Force
+        }
+    }
+}
+
 Write-Step "Deploy complete."
 Write-Step "If workers have cached the old plugin, restart them or submit a new job so Deadline copies the updated plugin sandbox."

scripts/farm_storage_remote_probe.py

@@ -0,0 +1,29 @@
+"""Second-host half of farm_storage_spike.py; use an interactive test account."""
+
+import json
+import os
+import sys
+import time
+from pathlib import Path
+
+from farm_storage_spike import atomic_json, share_lock, wait_file
+
+
+root = Path(sys.argv[1])
+try:
+    with share_lock(root):
+        atomic_json(root / "control.json", {"generation": 1, "check": "1" * 128})
+        atomic_json(root / "remote-ready.json", {"host": os.environ["COMPUTERNAME"], "pid": os.getpid()})
+        wait_file(root / "release-remote")
+    for generation in range(2, 102):
+        with share_lock(root):
+            atomic_json(root / "control.json", {"generation": generation, "check": str(generation) * 128})
+        time.sleep(0.01)
+    (root / "remote-writer-done").touch()
+    with share_lock(root):
+        (root / "remote-death-lock-held").touch()
+        wait_file(root / "kill-remote")
+        os._exit(23)
+except Exception as exc:
+    (root / "remote-error.txt").write_text(str(exc), encoding="utf-8")
+    raise

scripts/install_native_session.ps1

@@ -0,0 +1,56 @@
+<# Install a worker-local lifetime owner outside Deadline's render process tree.
+Warm mode is permitted only when the Worker/GPU is reserved for ComfyUI. #>
+[CmdletBinding(SupportsShouldProcess=$true)]
+param(
+    [Parameter(Mandatory=$true)][string]$Worker,
+    [Parameter(Mandatory=$true)][string]$GpuUuid,
+    [Parameter(Mandatory=$true)][string]$ComfyRoot,
+    [string]$StateRoot = "$env:LOCALAPPDATA\ComfyUI-Deadline\native-session",
+    [int]$BrokerPort = 8202,
+    [int]$ComfyPort = 8198,
+    [int]$IdleSeconds = 60,
+    [ValidateSet('none','auto','latent2rgb','taesd')][string]$PreviewMethod = 'auto',
+    [switch]$ExclusiveWorker,
+    [string]$BorrowedUrl = "",
+    [string]$BorrowedInputRoot = "",
+    [string]$BorrowedOutputRoot = "",
+    [string]$BorrowedTempRoot = "",
+    [string[]]$CustomNodes = @()
+)
+$ErrorActionPreference = 'Stop'
+if (!$ExclusiveWorker -and !$BorrowedUrl) { throw 'Persistent warm owners require -ExclusiveWorker on a ComfyUI-only GPU. Normal render tasks need no installed owner.' }
+if ($Worker -notmatch '^[A-Za-z0-9_.-]+$' -or $GpuUuid -notmatch '^GPU-[a-fA-F0-9-]+$') { throw 'Invalid Worker/GPU identity' }
+if ($BrokerPort -lt 1024 -or $BrokerPort -gt 65535 -or $ComfyPort -lt 1024 -or $ComfyPort -gt 65535) { throw 'Invalid port' }
+if ($IdleSeconds -lt 1 -or $IdleSeconds -gt 3600) { throw 'Idle timeout must be 1 to 3600 seconds' }
+$resolvedComfy = (Resolve-Path -LiteralPath $ComfyRoot).Path
+if (!(Test-Path -LiteralPath (Join-Path $resolvedComfy 'main.py'))) { throw 'ComfyRoot must contain main.py' }
+$python = Join-Path (Split-Path -Parent $resolvedComfy) 'python_embeded\python.exe'
+if (!(Test-Path -LiteralPath $python)) { throw 'Portable Python was not found' }
+$project = Split-Path -Parent $PSScriptRoot
+$owner = Join-Path $project 'farm_session_owner.py'
+$taskName = "ComfyUI-NativeQueue-$Worker-$BrokerPort"
+$existing = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
+if ($existing -and $existing.State -eq 'Running') { throw "Drain/stop the existing owner before updating $taskName" }
+# Quotes are literal argument separators for Windows process creation.
+foreach ($value in @($resolvedComfy,$StateRoot,$BorrowedUrl,$BorrowedInputRoot,$BorrowedOutputRoot,$BorrowedTempRoot) + $CustomNodes) {
+    if ($value.Contains('"') -or $value.Contains("`n") -or $value.Contains("`r")) { throw 'Invalid quoted argument' }
+}
+$arguments = '-s "' + $owner + '" --worker "' + $Worker + '" --gpu-uuid "' + $GpuUuid + '" --comfy-root "' + $resolvedComfy + '" --state-root "' + $StateRoot + '" --broker-port ' + $BrokerPort + ' --comfy-port ' + $ComfyPort + ' --idle-seconds ' + $IdleSeconds
+$arguments += ' --preview-method ' + $PreviewMethod
+if ($ExclusiveWorker) { $arguments += ' --exclusive-worker' }
+if ($BorrowedUrl) {
+    if (!$BorrowedInputRoot -or !$BorrowedOutputRoot) { throw 'Borrowed sessions require their actual input and output folders' }
+    $arguments += ' --borrowed-url "' + $BorrowedUrl + '" --borrowed-input-root "' + $BorrowedInputRoot + '" --borrowed-output-root "' + $BorrowedOutputRoot + '"'
+    if ($BorrowedTempRoot) { $arguments += ' --borrowed-temp-root "' + $BorrowedTempRoot + '"' }
+}
+if ($CustomNodes.Count) { $arguments += ' --custom-nodes ' + (($CustomNodes | ForEach-Object { '"' + $_ + '"' }) -join ' ') }
+if ($PSCmdlet.ShouldProcess($taskName, 'Register and start native session owner')) {
+    $action = New-ScheduledTaskAction -Execute $python -Argument $arguments -WorkingDirectory $resolvedComfy
+    $principal = New-ScheduledTaskPrincipal -UserId ([Security.Principal.WindowsIdentity]::GetCurrent().Name) -LogonType Interactive -RunLevel Limited
+    $trigger = New-ScheduledTaskTrigger -AtLogOn -User $principal.UserId
+    $settings = New-ScheduledTaskSettingsSet -Hidden -ExecutionTimeLimit ([TimeSpan]::Zero) -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 1)
+    Register-ScheduledTask -TaskName $taskName -Action $action -Principal $principal -Trigger $trigger -Settings $settings -Force | Out-Null
+    Start-ScheduledTask -TaskName $taskName
+    Write-Host "Installed $taskName. Configure NativeWorkerSessions in Deadline's ComfyUI plugin."
+    if (!$ExclusiveWorker -and !$BorrowedUrl) { Write-Host 'Shared Worker mode: GPU/process is released before each task completes.' }
+}

scripts/validate_native_queue_live.py

@@ -0,0 +1,123 @@
+"""Isolated live durability gate on the current M28 Worker.
+
+Starts with Worker offline, restricts it to a unique test user, submits three
+accepted jobs, kills only the CPU test submitter, then verifies durable results.
+Run after starting the isolated submitter on 8197 and the session owner on 8202.
+"""
+import argparse
+import json
+import subprocess
+import sys
+import time
+import urllib.request
+import uuid
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+from farm_backend import deadline, parse_ini
+from farm_protocol import RunStore, atomic_json, read_json
+
+
+def request(url, body=None):
+    try:
+        with urllib.request.urlopen(urllib.request.Request(url, data=json.dumps(body).encode() if body is not None else None,
+                headers={"Content-Type": "application/json"}), timeout=90) as response:
+            return json.load(response)
+    except urllib.error.HTTPError as exc:
+        raise RuntimeError(exc.read().decode()) from exc
+
+
+def payload(batch):
+    return {"submission_key": str(uuid.uuid4()), "batch_id": batch,
+        "prompt": {"1": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}},
+            "2": {"class_type": "EmptySD3LatentImage", "inputs": {"width": 64, "height": 64, "batch_size": 1}},
+            "3": {"class_type": "VAEDecode", "inputs": {"samples": ["2", 0], "vae": ["1", 0]}},
+            "4": {"class_type": "SaveImage", "inputs": {"images": ["3", 0], "filename_prefix": "native-test"}}},
+        "extra_data": {"extra_pnginfo": {"workflow": {"id": batch, "nodes": [], "links": [], "version": .4}}}}
+
+
+def run(args):
+    store = RunStore(args.root)
+    args.root.mkdir(parents=True, exist_ok=True)
+    if (args.root / "validation.json").exists():
+        raise ValueError("Use a fresh root")
+    if deadline("-GetSlaveInfo", args.worker, "SlaveStatus").strip() != "Offline":
+        raise ValueError("The test Worker must be offline")
+    backup = {key: deadline("-GetSlaveSetting", args.worker, key).strip()
+        for key in ("SchedulingMode", "UserJobsModeNames", "ConcurrentTasksLimit")}
+    atomic_json(args.root / "worker_settings_before.json", backup)
+    scope = "native_queue_live_" + uuid.uuid4().hex[:8]
+    jobs, runs = [], []
+    try:
+        for _ in range(20):
+            active = request(args.submitter + "/api/jobs?status=pending,in_progress")["jobs"]
+            if not active: break
+            time.sleep(1)
+        if active: raise ValueError("Finish/cancel previous isolated runs before switching test roots")
+        deadline("-SetSlaveSetting", args.worker, "UserJobsModeNames", scope)
+        deadline("-SetSlaveSetting", args.worker, "SchedulingMode", "UserJobs")
+        deadline("-SetSlaveSetting", args.worker, "ConcurrentTasksLimit", "1")
+        settings = request(args.submitter + "/deadline/farm/settings")["settings"]
+        request(args.submitter + "/deadline/farm/settings", {**settings, "enabled": True, "root": str(args.root),
+            "scope": scope, "pool": "test", "workers": args.worker, "priority": 90})
+        batch = str(uuid.uuid4())
+        for _ in range(3):
+            body = payload(batch)
+            accepted = request(args.submitter + "/deadline/farm/submit", body)
+            run_id = accepted["prompt_id"]
+            # A lost transport acknowledgement retries exactly this durable key.
+            assert request(args.submitter + "/deadline/farm/submit", body) == accepted
+            runs.append(run_id)
+            jobs.append(read_json(store.run(run_id) / "receipt.json")["job_id"])
+            atomic_json(args.root / "runs.json", {"runs": runs, "jobs": jobs, "scope": scope})
+        identity = request(args.submitter + "/deadline/session")
+        assert identity["pid"] != request("http://127.0.0.1:8188/deadline/session")["pid"]
+        subprocess.run(["powershell.exe", "-NoProfile", "-Command", "Stop-Process -Id " + str(identity["pid"])], check=True)
+        print("Accepted all runs; isolated submitting ComfyUI stopped: " + str(identity["pid"]), flush=True)
+        subprocess.run(["powershell.exe", "-NoProfile", "-Command", "Start-ScheduledTask -TaskName 'ComfyUI-NativeQueue-WorkerSpike'"], check=True)
+        until, previous = time.monotonic() + 240, None
+        while time.monotonic() < until:
+            states = parse_ini(deadline("-GetJob", ",".join(jobs), "True"))
+            status = [states[j]["Status"] for j in jobs]
+            if previous != status:
+                print("Deadline: " + ", ".join(status), flush=True)
+                previous = status
+            if all(s == "Completed" for s in status):
+                results = [store.result(i) for i in runs]
+                if not all(results): raise RuntimeError("Completed without a committed result")
+                controls = [store.control(i) for i in runs]
+                report = {"runs": runs, "jobs": jobs, "scope": scope, "submitter_pid_stopped": identity["pid"],
+                    "worker_pids": [s["worker_pid"] for s in controls], "sessions": [s["session_id"] for s in controls],
+                    "result_files": [len(r["files"]) for r in results], "submitter_off_completion": True,
+                    "cross_job_warmth": len({s["session_id"] for s in controls}) == 1}
+                atomic_json(args.root / "validation.json", report)
+                print(json.dumps(report, indent=2), flush=True)
+                return report
+            if any(s == "Failed" for s in status) or any(args.worker in s.get("BadSlaves", "").split(",") for s in states.values()):
+                raise RuntimeError("Deadline excluded/failed the test Worker; inspect task reports")
+            time.sleep(3)
+        raise TimeoutError("Durable live gate timed out")
+    finally:
+        if jobs:
+            states = parse_ini(deadline("-GetJob", ",".join(jobs), "True"))
+            active = [j for j in jobs if states[j]["Status"] == "Active"]
+            if active: deadline("-SuspendJob", ",".join(active))
+        worker = Path(r"C:\Program Files\Thinkbox\Deadline10\bin\deadlineworker.exe")
+        subprocess.run([str(worker), "-shutdown", "-name", args.worker], capture_output=True, timeout=30,
+            creationflags=subprocess.CREATE_NO_WINDOW)
+        for _ in range(30):
+            if deadline("-GetSlaveInfo", args.worker, "SlaveStatus").strip() == "Offline": break
+            time.sleep(1)
+        if deadline("-GetSlaveInfo", args.worker, "SlaveStatus").strip() != "Offline":
+            raise RuntimeError("Worker not offline; restore settings only after it stops")
+        for key, value in backup.items(): deadline("-SetSlaveSetting", args.worker, key, value)
+        print("Worker stopped; original settings restored", flush=True)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--root", type=Path, required=True)
+    parser.add_argument("--worker", default="M28")
+    parser.add_argument("--submitter", default="http://127.0.0.1:8197")
+    run(parser.parse_args())

tests/test_deadline_discovery.mjs

@@ -0,0 +1,27 @@
+import {test} from "node:test";
+import assert from "node:assert/strict";
+import {fetchFarmDiscovery, farmConnectionStatus} from "../web/deadline_sidebar.js";
+
+test("missing discovery route identifies an old backend instead of farm disconnection", async () => {
+    const api = {async fetchApi() {return new Response("Not found",{status:404});}};
+    await assert.rejects(fetchFarmDiscovery(api,false), error => error.restartRequired && /Restart ComfyUI/.test(error.message));
+    assert.equal(farmConnectionStatus({restart_required:true,connected:false},{enabled:false}),"Restart required");
+});
+test("repository connectivity is independent of submission being enabled", () => {
+    assert.equal(farmConnectionStatus({connected:true},{enabled:false},{connected:false}),"Connected");
+    assert.equal(farmConnectionStatus({connected:true},{enabled:true},{connected:false}),"Connected");
+    assert.equal(farmConnectionStatus({connected:false},{enabled:true},{connected:true}),"Disconnected");
+});
+test("real discovery failures retain their connection diagnostic", async () => {
+    const api = {async fetchApi() {return new Response("Unavailable",{status:503});}};
+    await assert.rejects(fetchFarmDiscovery(api,false), error => !error.restartRequired && /503/.test(error.message));
+});
+test("successful discovery passes through data, refresh flag and abort signal", async () => {
+    const signal=new AbortController().signal;
+    const data={connected:true,workers:[{name:"M28"}],pools:["test"]};
+    const api={async fetchApi(route,options) {
+        assert.equal(route,"/deadline/farm/discovery?refresh=1");assert.equal(options.signal,signal);
+        return new Response(JSON.stringify(data));
+    }};
+    assert.deepEqual(await fetchFarmDiscovery(api,true,signal),data);
+});

tests/test_deadline_queue.mjs

@@ -0,0 +1,241 @@
+import {test} from "node:test";
+import assert from "node:assert/strict";
+import {FarmProgress, installActivePagination, installRouting} from "../web/deadline_routing.js";
+import {captureDeadlineDestination} from "../web/native_queue_compat.js";
+
+function fixture() {
+    const calls = [];
+    const args = [];
+    const api = {
+        dispatchCustomEvent() {},
+        async fetchApi(route, options) {
+            calls.push({route, body: JSON.parse(options.body)});
+            return new Response(JSON.stringify({prompt_id: crypto.randomUUID(), node_errors: {}}));
+        },
+        async queuePrompt(number, data, options) {
+            args.push([number, options]);
+            const response = await this.fetchApi("/prompt", {method: "POST", body: JSON.stringify({
+                prompt: data.output, extra_data: {auth_token_comfy_org: "private", api_key_comfy_org: "private",
+                    extra_pnginfo: {workflow: data.workflow}}, partial_execution_targets: options?.partialExecutionTargets,
+                front: number === -1
+            })});
+            return await response.json();
+        }
+    };
+    let seed = 40;
+    const app = {
+        nextQueueRequestId: 1, queueItems: [], processingQueue: false,
+        async graphToPrompt() { return {output: {"1": {class_type: "KSampler", inputs: {seed: seed++}}}, workflow: {id: "workflow"}}; },
+        async queuePrompt(number, count = 1, options) {
+            this.queueItems.push({number, count, options, requestId: this.nextQueueRequestId++});
+            if (this.processingQueue) return false;
+            this.processingQueue = true;
+            try {
+                while (this.queueItems.length) {
+                    const item = this.queueItems.pop();
+                    for (let i = 0; i < item.count; i++) {
+                        const data = await this.graphToPrompt();
+                        await api.queuePrompt(item.number, data, item.options);
+                    }
+                }
+            } finally { this.processingQueue = false; }
+            return true;
+        }
+    };
+    return {api, app, calls, args};
+}
+
+test("finite native batch freezes destination, settings and seeds and preserves wrappers/options", async () => {
+    const f = fixture();
+    let destination = {settings: {pool: "test", priority: 50}};
+    const restore = installRouting(f.app, f.api, () => destination);
+    const queue = f.api.queuePrompt;
+    let wrapperCalls = 0;
+    f.api.queuePrompt = function (...args) { wrapperCalls++; return queue.apply(this, args); };
+    const original = f.api.fetchApi;
+    f.api.fetchApi = function (...args) {
+        if (destination) destination.settings.pool = "changed";
+        destination = null;
+        return original.apply(this, args);
+    };
+    const options = {previewMethod: "latent2rgb", arbitraryOption: true};
+    await f.app.queuePrompt(-1, 3, options);
+    assert.equal(wrapperCalls, 3);
+    assert.deepEqual(f.calls.map(call => call.route), Array(3).fill("/deadline/farm/submit"));
+    assert.deepEqual(f.calls.map(call => call.body.prompt["1"].inputs.seed), [40, 41, 42]);
+    assert.equal(new Set(f.calls.map(call => call.body.batch_id)).size, 1);
+    assert.equal(new Set(f.calls.map(call => call.body.submission_key)).size, 3);
+    for (const call of f.calls) {
+        assert.equal(call.body.settings.pool, "test");
+        assert.equal(call.body.front, undefined);
+        assert.equal(call.body.extra_data.auth_token_comfy_org, undefined);
+        assert.equal(call.body.extra_data.api_key_comfy_org, undefined);
+        assert.equal(call.body.extra_data.extra_pnginfo.workflow.__deadline_native_queue, undefined);
+    }
+    assert.ok(f.args.every(([number, opts]) => number === -1 && opts === options));
+    restore();
+});
+
+test("unready selected farm stops before widgets, transport, and local fallback", () => {
+    const f = fixture();
+    installRouting(f.app, f.api, () => captureDeadlineDestination(true, "disabled", {enabled:true},
+        {ready:false,message:"Selected Worker needs native session setup."}));
+    assert.throws(() => f.app.queuePrompt(0, 2), /Selected Worker needs native session setup/);
+    assert.equal(f.app.nextQueueRequestId, 1);
+    assert.equal(f.app.queueItems.length, 0);
+    assert.equal(f.calls.length, 0);
+});
+
+test("farm readiness does not prevent explicit local Run", async () => {
+    const f = fixture();
+    installRouting(f.app, f.api, () => captureDeadlineDestination(false, "disabled", {}, {ready:false}));
+    await f.app.queuePrompt(0, 1);
+    assert.deepEqual(f.calls.map(call => call.route), ["/prompt"]);
+});
+
+test("native automatic queue modes exclude Deadline without consulting readiness", () => {
+    for (const mode of ["change", "instant"]) {
+        assert.equal(captureDeadlineDestination(true, mode, {enabled:true}, {ready:false}), null);
+    }
+});
+
+test("Distributed nodes cannot reach browser dispatch when Deadline is selected", async () => {
+    for (const class_type of ["DistributedCollector", "DeadlineDistributedCollector", "DistributedVideoCollector",
+        "DeadlineDistributedVideoCollector", "UltimateSDUpscaleDistributed", "DeadlineUltimateSDUpscaleDistributed"]) {
+        const f = fixture();
+        f.app.graphToPrompt = async () => ({output:{"1":{class_type,inputs:{}}},workflow:{id:"workflow"}});
+        let dispatched = 0;
+        f.api.queuePrompt = async () => { dispatched++; };
+        installRouting(f.app, f.api, () => ({settings:{enabled:true}}));
+        await assert.rejects(f.app.queuePrompt(0, 1), /Use local Run.*Distributed/);
+        assert.equal(dispatched, 0);
+        assert.equal(f.calls.length, 0);
+    }
+});
+
+test("local Distributed workflows retain their existing dispatcher", async () => {
+    const f = fixture();
+    f.app.graphToPrompt = async () => ({output:{"1":{class_type:"DistributedCollector",inputs:{}}},workflow:{id:"workflow"}});
+    let dispatched = 0;
+    f.api.queuePrompt = async () => { dispatched++; };
+    installRouting(f.app, f.api, () => null);
+    await f.app.queuePrompt(0, 1);
+    assert.equal(dispatched, 1);
+});
+
+test("ready farm captures settings once and later readiness loss stops another batch", async () => {
+    const f = fixture(), settings = {enabled:true,pool:"test"};
+    let ready = true;
+    installRouting(f.app, f.api, () => captureDeadlineDestination(true, "disabled", settings,
+        {ready,message:"Deadline connection is unavailable."}));
+    const captured = captureDeadlineDestination(true, "disabled", settings, {ready:true});
+    settings.pool = "updated";
+    assert.equal(captured.settings.pool, "test");
+    await f.app.queuePrompt(0, 2);
+    ready = false;
+    assert.throws(() => f.app.queuePrompt(0, 1), /Deadline connection is unavailable/);
+    assert.deepEqual(f.calls.map(call => call.route), ["/deadline/farm/submit", "/deadline/farm/submit"]);
+});
+
+test("interleaved local and farm queue requests keep their own destination", async () => {
+    const f = fixture();
+    let farm = true;
+    installRouting(f.app, f.api, () => farm ? {settings: {}} : null);
+    const first = f.app.queuePrompt(0, 2);
+    farm = false;
+    const second = f.app.queuePrompt(0, 1);
+    await Promise.all([first, second]);
+    assert.deepEqual(f.calls.map(call => call.route), ["/deadline/farm/submit", "/deadline/farm/submit", "/prompt"]);
+    await f.api.queuePrompt(0, await f.app.graphToPrompt());
+    assert.equal(f.calls.at(-1).route, "/prompt");
+});
+
+test("partial batch transport failure stops further submissions without local fallback", async () => {
+    const f = fixture();
+    const fetch = f.api.fetchApi;
+    let requests = 0;
+    f.api.fetchApi = function (...args) {
+        if (++requests === 2) throw new Error("transport lost");
+        return fetch.apply(this, args);
+    };
+    installRouting(f.app, f.api, () => ({settings: {pool: "test"}}));
+    await assert.rejects(f.app.queuePrompt(0, 3), /transport lost/);
+    assert.equal(f.calls.length, 1);
+    assert.equal(f.calls[0].route, "/deadline/farm/submit");
+    const firstKey = f.calls[0].body.submission_key;
+    await f.app.queuePrompt(0, 1);
+    assert.equal(f.calls[1].route, "/deadline/farm/submit");
+    assert.notEqual(f.calls[1].body.submission_key, firstKey);
+});
+
+test("partial runs are rejected before native widget callbacks", () => {
+    const f = fixture();
+    installRouting(f.app, f.api, () => ({settings: {}}));
+    assert.throws(() => f.app.queuePrompt(0, 1, {queueNodeIds: ["1"]}), /whole workflows/);
+    assert.equal(f.app.nextQueueRequestId, 1);
+    assert.equal(f.calls.length, 0);
+});
+
+test("farm submission requires a finite bounded native batch", () => {
+    const f = fixture();
+    installRouting(f.app, f.api, () => ({settings: {}}));
+    for (const count of [0, -1, Infinity, NaN, 10001, 1.5]) assert.throws(() => f.app.queuePrompt(0, count), /finite batch/);
+    assert.equal(f.calls.length, 0);
+});
+
+test("extensions that captured an early API method cannot bypass farm routing", async () => {
+    const f = fixture();
+    const capturedEarly = f.api.queuePrompt.bind(f.api);
+    installRouting(f.app, f.api, () => ({settings: {pool: "test"}}));
+    f.api.queuePrompt = (...args) => capturedEarly(...args);
+    await f.app.queuePrompt(0, 2);
+    assert.deepEqual(f.calls.map(call => call.route), ["/deadline/farm/submit", "/deadline/farm/submit"]);
+});
+
+test("Submit to Deadline nodes cannot recursively submit or fall back to local execution", async () => {
+    const f = fixture();
+    f.app.graphToPrompt = async () => ({output: {"1": {class_type: "DeadlineSubmit", inputs: {}}}, workflow: {}});
+    installRouting(f.app, f.api, () => ({settings: {}}));
+    await f.app.queuePrompt(0);
+    assert.equal(f.calls.length, 0);
+});
+
+test("active queue loads beyond 200 without a false active count", async () => {
+    const calls = [];
+    const api = {async fetchApi(route) {
+        calls.push(route);
+        const offset = Number(new URL(route, "http://local").searchParams.get("offset"));
+        const jobs = Array.from({length: offset === 0 ? 200 : 7}, (_, index) => ({id: String(offset + index)}));
+        return new Response(JSON.stringify({jobs, pagination: {offset, limit: 200, total: 207, has_more: offset === 0}}));
+    }};
+    const restore = installActivePagination(api);
+    const data = await (await api.fetchApi("/jobs?status=in_progress,pending&limit=200&offset=0")).json();
+    assert.equal(data.jobs.length, 207);
+    assert.equal(data.pagination.total, 207);
+    assert.equal(calls.length, 2);
+    restore();
+});
+
+test("progress fences late generations and duplicate callbacks per run", () => {
+    const progress = new FarmProgress();
+    const event = {run_id: "a", generation: 2, sequence: 3, nodes: {"1": {value: 50}}};
+    assert.equal(progress.apply(event), true);
+    assert.equal(progress.apply({...event}), false);
+    assert.equal(progress.apply({...event, generation: 1, sequence: 99}), false);
+    assert.equal(progress.apply({...event, run_id: "b", nodes: {"1": {value: 80}}}), true);
+    assert.equal(progress.runs.get("a").nodes["1"].value, 50);
+    assert.equal(progress.runs.get("b").nodes["1"].value, 80);
+    progress.restore([{...event, sequence: 4}]);
+    assert.equal(progress.runs.has("b"), false);
+});
+
+test("queued Stop advances through cancellation intent and terminal acknowledgement", () => {
+    const progress = new FarmProgress();
+    const queued = {run_id:"queued", generation:0, sequence:0, status:"pending", cancel_requested:false, nodes:{}};
+    assert.equal(progress.apply(queued), true);
+    assert.equal(progress.apply({...queued, sequence:1, cancel_requested:true}), true);
+    assert.equal(progress.runs.get("queued").cancel_requested, true);
+    assert.equal(progress.apply(queued), false);
+    assert.equal(progress.apply({...queued, sequence:2, status:"cancelled", cancel_requested:true}), true);
+    assert.equal(progress.runs.get("queued").status, "cancelled");
+});

tests/test_deadline_settings.mjs

@@ -0,0 +1,125 @@
+import {test} from "node:test";
+import assert from "node:assert/strict";
+import {DeadlineSettings} from "../web/deadline_settings.js";
+
+function fixture() {
+    let server = {enabled: false, pool: "test", workers: "M28", priority: 50};
+    let fail = false;
+    const api = {async fetchApi(route, options = {}) {
+        if (options.method === "POST") {
+            const body = JSON.parse(options.body);
+            assert.deepEqual(body.expected_settings, server);
+            if (fail) return new Response(JSON.stringify({error: {message: "Share unavailable"}}), {status:503});
+            server = body.settings; return new Response(JSON.stringify(server));
+        }
+        return new Response(JSON.stringify({settings:server, health:{connected:true}}));
+    }};
+    const saved = [];
+    return {model:new DeadlineSettings(api, value => saved.push(value)), saved,
+        external(value) {server = {...server, ...value};}, fail() {fail = true;}};
+}
+test("refresh preserves unsaved settings and missing Worker names", async () => {
+    const {model} = fixture(); await model.load(); model.edit("workers", "M28,RemovedWorker"); model.edit("priority", 85);
+    await model.load(); assert.equal(model.draft.priority, 85); assert.equal(model.draft.workers, "M28,RemovedWorker");
+    assert.equal(model.conflict, false); assert.equal(model.dirty, true);
+    model.revert(); assert.equal(model.draft.workers, "M28"); assert.equal(model.dirty, false);
+});
+test("external edits require explicit revert before save", async () => {
+    const f = fixture(); await f.model.load(); f.model.edit("priority", 85); f.external({priority:60});
+    await f.model.load(); await f.model.load(); assert.equal(f.model.conflict, true);
+    assert.equal(await f.model.save(), false); assert.equal(f.model.draft.priority,85);
+    f.model.revert(); assert.equal(f.model.draft.priority,60); assert.equal(f.model.conflict,false);
+});
+test("Apply persists one atomic draft and failed saves preserve it", async () => {
+    const f = fixture(); await f.model.load(); f.model.edit("priority",90);
+    assert.equal(await f.model.save(),true); assert.equal(f.model.dirty,false); assert.equal(f.saved.at(-1).priority,90);
+    f.model.edit("priority",80); f.fail(); assert.equal(await f.model.save(),false);
+    assert.equal(f.model.draft.priority,80); assert.equal(f.model.saved.priority,90); assert.match(f.model.error,/Share unavailable/);
+});
+test("closing a sidebar aborts late settings updates", async () => {
+    const f = fixture(); const controller = new AbortController(); controller.abort();
+    await assert.rejects(f.model.load(controller.signal), /aborted/); assert.equal(f.model.saved,null);
+});
+test("a save conflict reloads the authoritative settings before Revert", async () => {
+    let server = {priority:50};
+    const model = new DeadlineSettings({async fetchApi(route, options = {}) {
+        return options.method === "POST" ? new Response(JSON.stringify({error:{type:"deadline_settings_conflict",message:"Changed elsewhere"}}),{status:409}) :
+            new Response(JSON.stringify({settings:server}));
+    }},()=>{});
+    await model.load(); model.edit("priority",90); server = {priority:60};
+    assert.equal(await model.save(),false); assert.equal(model.conflict,true); assert.equal(model.draft.priority,90);
+    model.revert(); assert.equal(model.draft.priority,60);
+});
+
+function deferred() {
+    let resolve;
+    const promise = new Promise(done => { resolve = done; });
+    return {promise, resolve};
+}
+
+test("a refresh started before Apply cannot restore old saved settings or routing", async () => {
+    const oldRead = deferred();
+    let reads = 0, server = {priority: 50};
+    const routed = [];
+    const model = new DeadlineSettings({async fetchApi(route, options = {}) {
+        if (options.method === "POST") {
+            server = JSON.parse(options.body).settings;
+            return new Response(JSON.stringify(server));
+        }
+        const snapshot = {...server};
+        if (++reads === 2) await oldRead.promise;
+        return new Response(JSON.stringify({settings: snapshot}));
+    }}, settings => routed.push(settings));
+    await model.load();
+    model.edit("priority", 90);
+    const refresh = model.load();
+    assert.equal(await model.save(), true);
+    oldRead.resolve();
+    await refresh;
+    assert.equal(server.priority, 90);
+    assert.equal(model.saved.priority, 90);
+    assert.equal(model.draft.priority, 90);
+    assert.equal(model.dirty, false);
+    assert.equal(routed.at(-1).priority, 90);
+});
+
+test("a refresh started during Apply cannot overwrite the committed response", async () => {
+    const post = deferred(), oldRead = deferred();
+    let reads = 0, server = {priority: 50};
+    const model = new DeadlineSettings({async fetchApi(route, options = {}) {
+        if (options.method === "POST") {
+            await post.promise;
+            server = JSON.parse(options.body).settings;
+            return new Response(JSON.stringify(server));
+        }
+        const snapshot = {...server};
+        if (++reads === 2) await oldRead.promise;
+        return new Response(JSON.stringify({settings: snapshot}));
+    }}, () => {});
+    await model.load();
+    model.edit("priority", 90);
+    const save = model.save();
+    const refresh = model.load();
+    post.resolve();
+    assert.equal(await save, true);
+    oldRead.resolve();
+    await refresh;
+    assert.equal(model.saved.priority, 90);
+    assert.equal(model.conflict, false);
+});
+
+test("out-of-order refreshes accept only the latest request", async () => {
+    const oldRead = deferred();
+    let reads = 0;
+    const model = new DeadlineSettings({async fetchApi() {
+        const priority = ++reads === 1 ? 50 : 60;
+        if (reads === 1) await oldRead.promise;
+        return new Response(JSON.stringify({settings: {priority}}));
+    }}, () => {});
+    const oldRefresh = model.load();
+    await model.load();
+    oldRead.resolve();
+    await oldRefresh;
+    assert.equal(model.saved.priority, 60);
+    assert.equal(model.draft.priority, 60);
+});

tests/test_deadline_sidebar.mjs

@@ -0,0 +1,39 @@
+import {test} from "node:test";
+import assert from "node:assert/strict";
+import {sidebarReadiness} from "../web/deadline_sidebar.js";
+import {evaluateWorkers} from "../web/deadline_workers.js";
+const settings = {root:"shared-root",enabled:true,workers:"",pool:"test",group:"comfy"};
+const discovery = {connected:true,plugin_available:true,workers:[]};
+function state(config = settings, farm = discovery, server = {ready:true}, dirty = false) {
+    return sidebarReadiness(config, farm, evaluateWorkers(config, farm), server, dirty);
+}
+test("pool/group scheduling needs no Worker allowlist or owner installation", () => {
+    assert.equal(state().ready, true);
+    assert.match(state().message, /native Run/);
+});
+test("deployment problems direct artists to the administrator guide", () => {
+    for (const result of [state({...settings, root:""}), state({...settings, enabled:false}), state(settings, {...discovery, plugin_available:false})]) {
+        assert.equal(result.action, "guide");
+        assert.equal(result.ready, undefined);
+        assert.doesNotMatch(result.message, /Set up this machine/);
+    }
+});
+test("failed storage verification cannot be hidden by a nonempty path", () => {
+    const result = state(settings, discovery, {ready:false,code:"storage",message:"Shared folder unavailable"});
+    assert.equal(result.ready, undefined);
+    assert.equal(result.message, "Shared folder unavailable");
+    assert.equal(result.action, "guide");
+});
+test("pending saves block submission until new settings are accepted", () => {
+    assert.equal(state(settings, discovery, {ready:true}, true).ready, undefined);
+});
+test("invalid optional restrictions remain visible and block submission", () => {
+    const result = state({...settings, workers:"Removed"});
+    assert.equal(result.ready, undefined);
+    assert.equal(result.action, "workers");
+    assert.match(result.message, /Removed/);
+});
+test("connection and restart diagnostics remain actionable", () => {
+    assert.equal(state(settings, {...discovery, connected:false}).action, "refresh");
+    assert.equal(state(settings, {...discovery, restart_required:true}).title, "Restart required");
+});

tests/test_deadline_workers.mjs

@@ -0,0 +1,43 @@
+import {test} from "node:test";
+import assert from "node:assert/strict";
+import {evaluateWorkers} from "../web/deadline_workers.js";
+const settings = {pool:"test", group:"comfy", workers:"", scope:"history"};
+const worker = (name, extra = {}) => ({name, host:name, enabled:true, status:"Idle", pools:["test"], groups:["comfy"], ...extra});
+const names = list => list.map(item => item.name);
+test("pool and group both constrain the optional list", () => {
+    const result = evaluateWorkers(settings, {workers:[worker("GPU"), worker("Pool", {pools:["other"]}), worker("Group", {groups:["other"]})]});
+    assert.deepEqual(names(result.candidates), ["GPU"]);
+});
+test("existing Workers need no warm-session mapping", () => {
+    assert.equal(evaluateWorkers(settings, {workers:[worker("Cold", {session_configured:false})]}).candidates[0].eligible, true);
+});
+test("missing and invalid selections survive search", () => {
+    const result = evaluateWorkers({...settings, workers:"Removed,Disabled,WrongPool,BadMapping"}, {workers:[
+        worker("Disabled", {enabled:false}), worker("WrongPool", {pools:["other"]}), worker("BadMapping", {session_error:"Invalid mapping"})]}, {search:"no-match"});
+    assert.equal(result.selected.length, 4);
+    assert.equal(result.counts.selectedEligible, 0);
+    assert.equal(result.candidates.length, 0);
+});
+test("names normalize and offline/busy Workers remain queueable", () => {
+    const result = evaluateWorkers({...settings, workers:"gpu,GPU,Offline"}, {workers:[worker("GPU", {status:"Rendering"}), worker("Offline", {status:"Offline"})]});
+    assert.equal(result.selected.length, 2);
+    assert.equal(result.counts.selectedEligible, 2);
+    assert.equal(result.counts.selectedAvailable, 0);
+});
+test("Deadline user determines eligibility independently of history scope", () => {
+    const result = evaluateWorkers(settings, {deadline_user:"artist", workers:[worker("Allowed", {scheduling_mode:"UserJobs", user_job_names:["ARTIST"]}), worker("Other", {scheduling_mode:"UserJobs", user_job_names:["other"]}), worker("Unknown", {scheduling_mode:"ListedJobs"})]});
+    assert.equal(result.counts.eligible, 1);
+    assert.equal(result.candidates.find(item => item.name === "Other").eligible, false);
+    assert.deepEqual(result.candidates.find(item => item.name === "Unknown").reasonCodes, ["scheduling"]);
+});
+test("machine scheduling requires a verified matching user", () => {
+    for (const machine_user of ["other", ""]) assert.equal(evaluateWorkers(settings, {deadline_user:"artist", workers:[worker("Machine", {scheduling_mode:"MachineJobs", machine_user})]}).counts.eligible, 0);
+});
+test("search keeps selected Workers and leaves discovery intact", () => {
+    const discovery = {workers:[worker("Selected"), worker("Remote", {host:"SIM01"})]};
+    const before = JSON.stringify(discovery);
+    const result = evaluateWorkers({...settings, workers:"Selected"}, discovery, {search:"SIM01"});
+    assert.deepEqual(names(result.selected), ["Selected"]);
+    assert.deepEqual(names(result.candidates), ["Remote"]);
+    assert.equal(JSON.stringify(discovery), before);
+});

tests/test_farm_backend.py

@@ -0,0 +1,551 @@
+import copy
+import concurrent.futures
+import contextlib
+import hashlib
+import json
+import tempfile
+import threading
+import types
+import unittest
+import uuid
+from pathlib import Path
+from unittest.mock import patch
+
+from farm_backend import FarmBackend, parse_ini, validate_settings, register_backend
+from farm_protocol import RunStore, read_json, atomic_json
+
+
+class FakeDeadline:
+    def __init__(self):
+        self.jobs = {}
+        self.submissions = 0
+        self.lose_submit = False
+        self.lose_resume = False
+        self.snapshot = {"repository": "\\\\repo\\repository", "plugin_available": True,
+            "workers": [{"name": "TestWorker", "enabled": True, "status": "Idle", "session_configured": True,
+                "pools": ["test"], "groups": ["none"]}], "pools": ["test"], "groups": ["none"]}
+
+    def __call__(self, *args):
+        if args[0] == "-ExecuteScript":
+            if args[1].endswith("deadline_job_status.py"):
+                return "COMFY_NATIVE_JOBS=" + json.dumps({i:{"Status":self.jobs[i]["Status"] if i in self.jobs else "Deleted"} for i in args[2:]})
+            return "COMFY_NATIVE_FARM=" + json.dumps(self.snapshot)
+        if args[0].endswith("job.info"):
+            self.submissions += 1
+            job_id = f"{self.submissions:024x}"
+            values = dict(line.split("=", 1) for line in Path(args[0]).read_text().splitlines())
+            self.jobs[job_id] = {"Status": "Suspended", "Name": values["Name"], "Plugin": values["Plugin"],
+                "run_id": values["ExtraInfoKeyValue1"].split("=", 1)[1]}
+            if self.lose_submit: raise TimeoutError("lost submit response")
+            return "JobID=" + job_id
+        if args[0] == "-GetJobsFilterIniAnd":
+            name = args[1].split("=", 1)[1]
+            ids = [i for i, j in self.jobs.items() if j["Name"] == name]
+        elif args[0] == "-GetJob":
+            ids = args[1].split(",")
+        elif args[0] == "-GetJobExtraInfoKeyValue": return self.jobs[args[1]]["run_id"]
+        elif args[0] == "-ResumeJob":
+            self.jobs[args[1]]["Status"] = "Active"
+            if self.lose_resume: raise TimeoutError("lost activation response")
+            return "Success"
+        elif args[0] == "-GetRepositoryPath": return "\\\\repo\\repository"
+        elif args[0] == "-GetCurrentUserName": return "artist"
+        else: raise AssertionError(args)
+        return "\n".join("[" + i + "]\n" + "\n".join(k + "=" + v for k, v in self.jobs[i].items()) for i in ids if i in self.jobs)
+
+
+class SubmissionTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.store = RunStore(self.root / "share")
+        self.store.initialize()
+        self.fake = FakeDeadline()
+        self.backend = FarmBackend(self.root / "settings.json", self.fake)
+        self.backend.settings.update(enabled=True, root="\\\\farm\\native", workers="TestWorker")
+        self.patches = [patch("farm_backend.RunStore", return_value=self.store),
+            patch.dict("sys.modules", {"folder_paths": types.SimpleNamespace(get_input_directory=lambda: str(self.root / "input"))})]
+        for p in self.patches: p.start(); self.addCleanup(p.stop)
+        self.body = {"prompt": {"1": {"class_type": "SaveImage", "inputs": {"seed": 123}}},
+            "extra_data": {"auth_token_comfy_org": "do-not-store", "extra_pnginfo": {"workflow": {"id": "workflow", "nodes": []}}},
+            "submission_key": str(uuid.uuid4()), "batch_id": str(uuid.uuid4())}
+
+    def test_transport_retry_and_distinct_clicks(self):
+        first = self.backend.submit(self.body)
+        self.assertEqual(self.backend.submit(self.body), first)
+        second = self.backend.submit({**self.body, "submission_key": str(uuid.uuid4())})
+        self.assertNotEqual(first["prompt_id"], second["prompt_id"])
+        self.assertEqual(self.fake.submissions, 2)
+        stored = read_json(self.store.run(first["prompt_id"]) / "submission.json")
+        self.assertNotIn("auth_token_comfy_org", stored["payload"]["extra_data"])
+        changed = copy.deepcopy(self.body)
+        changed["prompt"]["1"]["inputs"]["seed"] = 124
+        with self.assertRaisesRegex(ValueError, "different snapshot"): self.backend.submit(changed)
+        self.assertEqual(self.fake.submissions, 2)
+
+    def test_lost_submit_response_is_discovered_without_resubmit_after_restart(self):
+        self.fake.lose_submit = True
+        with self.assertRaises(TimeoutError): self.backend.submit(self.body)
+        self.fake.lose_submit = False
+        replacement = FarmBackend(self.root / "other-settings.json", self.fake)
+        replacement.settings = dict(self.backend.settings)
+        accepted = replacement.submit(self.body)
+        self.assertEqual(self.fake.submissions, 1)
+        self.assertTrue(read_json(self.store.run(accepted["prompt_id"]) / "receipt.json")["accepted"])
+
+    def test_lost_activation_response_does_not_duplicate_and_background_recovers(self):
+        self.fake.lose_resume = True
+        with self.assertRaises(TimeoutError): self.backend.submit(self.body)
+        self.fake.lose_resume = False
+        self.backend.refresh()
+        self.assertEqual(len(self.backend.jobs), 1)
+        self.assertTrue(next(iter(self.backend.jobs.values()))["deadline"]["accepted"])
+        self.assertEqual(self.fake.submissions, 1)
+
+    def test_unknown_submission_remains_unconfirmed_and_cannot_resubmit(self):
+        original = self.fake.__call__
+        def unavailable(*args):
+            if args[0].endswith("job.info"): raise TimeoutError("unknown")
+            return original(*args)
+        self.backend.command = unavailable
+        with self.assertRaises(TimeoutError): self.backend.submit(self.body)
+        self.backend.command = self.fake
+        with self.assertRaisesRegex(RuntimeError, "unconfirmed"): self.backend.submit(self.body)
+        self.assertEqual(self.fake.submissions, 0)
+
+    def test_partial_batch_failure_keeps_earlier_accepted_run(self):
+        first = self.backend.submit(self.body)["prompt_id"]
+        second = {**self.body, "submission_key": str(uuid.uuid4())}
+        self.fake.lose_submit = True
+        with self.assertRaises(TimeoutError): self.backend.submit(second)
+        self.assertTrue(read_json(self.store.run(first) / "receipt.json")["accepted"])
+        self.assertEqual(self.backend.jobs[first]["status"], "pending")
+        self.fake.lose_submit = False
+        recovered = self.backend.submit(second)["prompt_id"]
+        self.assertNotEqual(first, recovered)
+        self.assertEqual(self.fake.submissions, 2)
+
+    def test_retry_uses_original_resolved_snapshot(self):
+        original = self.backend.submit(self.body)["prompt_id"]
+        state = self.store.control(original)
+        state["status"] = "failed"
+        atomic_json(self.store.run(original) / "control.json", state)
+        self.fake.jobs[next(iter(self.fake.jobs))]["Status"] = "Failed"
+        self.backend.refresh()
+        retried = self.backend.retry(original, str(uuid.uuid4()))["prompt_id"]
+        stored = read_json(self.store.run(retried) / "submission.json")
+        self.assertEqual(stored["retry_of"], original)
+        self.assertEqual(stored["payload"]["prompt"]["1"]["inputs"]["seed"], 123)
+
+    def test_disabling_native_queue_blocks_retry_with_old_enabled_snapshot(self):
+        original = self.backend.submit(self.body)["prompt_id"]
+        self.backend.settings["enabled"] = False
+        with self.assertRaisesRegex(ValueError, "disabled"):
+            self.backend.retry(original, str(uuid.uuid4()))
+        self.assertEqual(self.fake.submissions, 1)
+
+    def test_settings_injection_and_local_storage_rejected(self):
+        with self.assertRaises(ValueError): validate_settings({"enabled": True, "root": "C:/local", "workers": "Test"})
+        with self.assertRaises(ValueError): validate_settings({"pool": "test\nPlugin=CommandLine"})
+        with self.assertRaises(ValueError): self.backend.submit({**self.body, "settings": {"scope": "someone-else"}})
+
+    def test_in_flight_submission_blocks_storage_change_without_holding_global_lock(self):
+        entered, release = threading.Event(), threading.Event()
+        original = self.backend._ensure_submitted
+        def blocked(store, submission):
+            entered.set()
+            if not release.wait(5):
+                raise TimeoutError("test did not release submission")
+            return original(store, submission)
+        with patch.object(self.backend, "_ensure_submitted", side_effect=blocked), concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+            pending = pool.submit(self.backend.submit, self.body)
+            try:
+                self.assertTrue(entered.wait(5))
+                self.assertEqual(self.backend.active_submissions, 1)
+                self.assertFalse(self.backend.jobs)
+                for changed in ({"root": "\\\\other\\native"}, {"scope": "other-user"}):
+                    with self.assertRaisesRegex(ValueError, "active submissions"):
+                        self.backend.configure({**self.backend.settings, **changed, "enabled": False})
+                # Existing requests keep their snapshot while disabling new ones.
+                self.backend.configure({**self.backend.settings, "enabled": False, "priority": 60})
+            finally:
+                release.set()
+            accepted = pending.result(timeout=5)["prompt_id"]
+        self.assertEqual(self.backend.active_submissions, 0)
+        self.assertIn(accepted, self.backend.jobs)
+        self.assertEqual(self.backend.settings["root"], "\\\\farm\\native")
+        self.assertEqual(self.backend.settings["priority"], 60)
+        with self.assertRaisesRegex(ValueError, "active submissions and runs"):
+            self.backend.configure({**self.backend.settings, "root": "\\\\other\\native"})
+
+    def test_failed_submission_releases_storage_change_guard(self):
+        with self.assertRaises(ValueError):
+            self.backend.submit({**self.body, "submission_key": "invalid"})
+        self.assertEqual(self.backend.active_submissions, 0)
+        self.backend.configure({**self.backend.settings, "enabled": False, "scope": "other-user"})
+        self.assertEqual(self.backend.settings["scope"], "other-user")
+
+    def test_lost_activation_response_blocks_storage_change_and_setup_without_cache(self):
+        self.fake.lose_resume = True
+        with self.assertRaises(TimeoutError):
+            self.backend.submit(self.body)
+        self.assertFalse(self.backend.jobs)
+        self.assertEqual(self.backend.active_submissions, 0)
+        with self.assertRaisesRegex(ValueError, "unconfirmed farm runs"):
+            self.backend.configure({**self.backend.settings, "enabled": False, "root": "\\\\other\\native"})
+        self.assertFalse(self.backend.configuration_in_progress)
+
+    def test_restart_checks_durable_active_run_before_changing_scope(self):
+        self.backend.submit(self.body)
+        restarted = FarmBackend(self.root / "restarted.json", self.fake)
+        restarted.settings = dict(self.backend.settings)
+        self.assertFalse(restarted.jobs)
+        with self.assertRaisesRegex(ValueError, "pending or unconfirmed farm runs"):
+            restarted.configure({**restarted.settings, "enabled": False, "scope": "other-user"})
+        self.assertEqual(restarted.settings["scope"], self.backend.settings["scope"])
+
+    def test_confirmed_terminal_deadline_run_does_not_block_scope_change_after_restart(self):
+        self.backend.submit(self.body)
+        self.fake.jobs[next(iter(self.fake.jobs))]["Status"] = "Completed"
+        restarted = FarmBackend(self.root / "restarted.json", self.fake)
+        restarted.settings = dict(self.backend.settings)
+        restarted.configure({**restarted.settings, "enabled": False, "scope": "other-user"})
+        self.assertEqual(restarted.settings["scope"], "other-user")
+
+    def test_storage_scan_blocks_new_capture_and_does_not_hold_global_lock(self):
+        entered, release = threading.Event(), threading.Event()
+        original = self.backend._check_durable_runs
+        def blocked(settings):
+            entered.set()
+            if not release.wait(5):
+                raise TimeoutError("test did not release storage scan")
+            return original(settings)
+        with patch.object(self.backend, "_check_durable_runs", side_effect=blocked), concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+            change = pool.submit(self.backend.configure, {**self.backend.settings, "enabled": False, "scope": "other-user"})
+            try:
+                self.assertTrue(entered.wait(5))
+                with self.assertRaisesRegex(ValueError, "configuration is changing"):
+                    self.backend.submit(self.body)
+            finally:
+                release.set()
+            change.result(timeout=5)
+        self.assertFalse(self.backend.configuration_in_progress)
+        self.assertEqual(self.fake.submissions, 0)
+
+    def test_receipt_recovery_is_counted_as_an_active_submission(self):
+        self.fake.lose_resume = True
+        with self.assertRaises(TimeoutError):
+            self.backend.submit(self.body)
+        self.fake.lose_resume = False
+        original = self.backend._ensure_submitted
+        def recovering(store, submission):
+            self.assertEqual(self.backend.active_submissions, 1)
+            with self.assertRaisesRegex(ValueError, "active submissions"):
+                self.backend.configure({**self.backend.settings, "enabled": False, "scope": "other-user"})
+            return original(store, submission)
+        with patch.object(self.backend, "_ensure_submitted", side_effect=recovering):
+            self.backend.refresh()
+        self.assertEqual(self.backend.active_submissions, 0)
+
+    def test_retry_snapshot_cannot_enter_another_storage_root(self):
+        captured = {**self.backend.settings, "root": "\\\\old\\native"}
+        with self.assertRaisesRegex(ValueError, "storage settings"):
+            self.backend.submit(self.body, captured=captured)
+        self.assertEqual(self.backend.active_submissions, 0)
+        self.assertEqual(self.fake.submissions, 0)
+
+    def test_submission_readiness_rejects_unusable_workers_before_writing_records(self):
+        for problem in ({"enabled": False}, {"session_error": "Invalid mapping"}, {"groups": ["other"]}):
+            with self.subTest(problem=problem):
+                self.fake.snapshot["workers"][0].update(enabled=True, session_configured=True, session_error=None, groups=["none"])
+                self.fake.snapshot["workers"][0].update(problem)
+                self.backend.settings["group"] = "none" if "groups" not in problem else "required"
+                self.backend.discovery_checked = 0
+                with self.assertRaises(ValueError):
+                    self.backend.submit(self.body)
+                self.assertEqual(list(self.store.records(self.backend.settings["scope"])), [])
+                self.assertEqual(self.fake.submissions, 0)
+                self.assertEqual(self.backend.active_submissions, 0)
+
+    def test_native_route_cannot_bypass_missing_plugin_readiness(self):
+        self.fake.snapshot["plugin_available"] = False
+        with self.assertRaisesRegex(ValueError, "native ComfyUI plugin"):
+            self.backend.submit(self.body)
+        self.assertEqual(self.fake.submissions, 0)
+        self.assertEqual(list(self.store.records(self.backend.settings["scope"])), [])
+        self.assertFalse(self.backend.readiness_cache["ready"])
+
+    def test_offline_mapped_worker_can_queue_and_effective_plugin_path_is_recorded(self):
+        self.fake.snapshot["workers"][0]["status"] = "Offline"
+        self.backend.settings["workers"] = "testworker"
+        self.backend.settings["plugin_directory"] = "\\\\farm\\native\\plugin"
+        accepted = self.backend.submit(self.body)["prompt_id"]
+        stored = read_json(self.store.run(accepted) / "submission.json")
+        self.assertEqual(stored["plugin_directory"], "\\\\farm\\native\\plugin")
+        job_info = (self.store.run(accepted) / "job.info").read_text()
+        self.assertIn("CustomPluginDirectory=\\\\farm\\native\\plugin", job_info)
+
+    def test_readiness_is_read_only_and_legacy_environment_plugin_is_checked(self):
+        self.backend.plugin_directory = "\\\\farm\\legacy\\plugins"
+        result = self.backend.readiness()
+        self.assertTrue(result["ready"])
+        self.assertFalse(self.backend.settings_path.exists())
+        self.assertEqual(list(self.store.records(self.backend.settings["scope"])), [])
+        self.assertEqual(self.fake.submissions, 0)
+
+    def test_unrestricted_cold_run_uses_real_deadline_user_and_no_machine_limit(self):
+        self.backend.settings["workers"] = ""
+        self.backend.settings["scope"] = "legacy-history"
+        self.fake.snapshot["workers"][0]["session_configured"] = False
+        accepted = self.backend.submit(self.body)["prompt_id"]
+        info = (self.store.run(accepted) / "job.info").read_text()
+        self.assertIn("UserName=artist\n", info)
+        self.assertNotIn("Whitelist=", info)
+        self.assertNotIn("MachineLimit=", info)
+        self.assertEqual(read_json(self.store.run(accepted) / "submission.json")["scope"], "legacy-history")
+
+    def test_priority_can_save_during_storage_outage_without_touching_the_share(self):
+        self.backend.health = {"connected":False,"message":"Share unavailable"}
+        with patch("farm_backend.RunStore") as store:
+            self.backend.configure({**self.backend.settings, "priority":75})
+        store.assert_not_called()
+        self.assertEqual(self.backend.settings["priority"], 75)
+        self.assertFalse(self.backend.readiness_cache["ready"])
+
+    def test_cancelled_cold_start_reconciles_after_deadline_stops_its_task(self):
+        run_id = self.backend.submit(self.body)["prompt_id"]
+        attempt_id = str(uuid.uuid4())
+        job_id = next(iter(self.fake.jobs))
+        state = self.store.begin_attempt(run_id, attempt_id, job_id, "TestWorker", {"worker":"TestWorker","start":1})
+        self.store.update_attempt(run_id, attempt_id, state["generation"], execution_mode="task-owned")
+        self.store.cancel(run_id)
+        # Killing a startup broker can raise a transport error in PreRenderTasks
+        # before Deadline finishes suspending its task. Cancel still wins.
+        self.store.update_attempt(run_id, attempt_id, state["generation"], status="failed")
+        self.fake.jobs[job_id]["Status"] = "Suspended"
+        def command(*args):
+            if args[0] == "-SuspendJob":
+                raise AssertionError("Already suspended jobs need no repeated suspend")
+            if args[0] == "-GetJobTask":
+                return "Status=Pending"
+            return self.fake(*args)
+        self.backend.command = command
+        self.backend.refresh()
+        self.assertEqual(self.store.control(run_id)["status"], "cancelled")
+        self.assertEqual(self.backend.jobs[run_id]["status"], "cancelled")
+
+    def test_deleted_cancelled_job_does_not_issue_control_commands(self):
+        run_id = self.backend.submit(self.body)["prompt_id"]
+        self.store.cancel(run_id)
+        self.fake.jobs.clear()
+        self.backend.refresh()
+        self.assertIn(run_id, self.backend.jobs)
+        self.assertTrue(self.store.control(run_id)["cancel_requested"])
+        self.assertEqual(self.backend.jobs[run_id]["status"], "cancelled")
+        self.assertEqual(self.backend.jobs[run_id]["deadline"]["status"], "Deleted")
+        self.assertEqual(self.store.control(run_id)["generation"], 1)
+        self.backend.refresh()
+        self.assertEqual(self.store.control(run_id)["generation"], 1)
+
+    def test_removed_job_fences_attempts_and_old_progress_without_resubmission(self):
+        run_id = self.backend.submit(self.body)["prompt_id"]
+        job_id = next(iter(self.fake.jobs))
+        attempt = str(uuid.uuid4())
+        state = self.store.begin_attempt(run_id, attempt, job_id, "TestWorker", {"worker":"TestWorker","start":1})
+        atomic_json(self.store.run(run_id) / "attempts" / attempt / "progress.json", {
+            "run_id":run_id,"generation":1,"sequence":100,"nodes":{"1":{"value":50,"max":100}},"preview_url":"/old-preview"})
+        self.fake.jobs.clear()
+        self.backend.refresh()
+        self.assertEqual(self.backend.jobs[run_id]["status"], "failed")
+        self.assertEqual(self.backend.jobs[run_id]["execution_error"]["exception_type"], "DeadlineJobRemoved")
+        self.assertEqual(self.backend.progress[0]["generation"], 2)
+        self.assertEqual(self.backend.progress[0]["nodes"], {})
+        self.assertNotIn("preview_url", self.backend.progress[0])
+        self.assertFalse(self.store.update_attempt(run_id, attempt, state["generation"], status="in_progress"))
+        self.assertFalse(self.store.commit(run_id, attempt, state["generation"], {"run_id":run_id,"attempt_id":attempt,"generation":1}))
+        self.assertIsNone(self.store.begin_attempt(run_id, str(uuid.uuid4()), job_id, "TestWorker", {"worker":"TestWorker","start":2}))
+        self.assertEqual(self.fake.submissions, 1)
+
+    def test_missing_job_confirmation_outage_retains_cached_state(self):
+        run_id = self.backend.submit(self.body)["prompt_id"]
+        self.backend.refresh()
+        cached = copy.deepcopy(self.backend.jobs)
+        self.fake.jobs.clear()
+        def command(*args):
+            if args[0] == "-ExecuteScript": raise TimeoutError("Repository unavailable")
+            return self.fake(*args)
+        self.backend.command = command
+        with self.assertRaises(TimeoutError): self.backend.refresh()
+        self.assertEqual(self.backend.jobs, cached)
+        self.assertEqual(self.store.control(run_id)["generation"], 0)
+        self.assertNotIn("deadline_removed", self.store.control(run_id))
+
+    def test_empty_bulk_response_cannot_override_fresh_existing_job(self):
+        run_id = self.backend.submit(self.body)["prompt_id"]
+        def command(*args):
+            if args[0] == "-GetJob": return ""
+            return self.fake(*args)
+        self.backend.command = command
+        self.backend.refresh()
+        self.assertEqual(self.backend.jobs[run_id]["status"], "pending")
+        self.assertNotIn("deadline_removed", self.store.control(run_id))
+
+    def test_deleted_job_after_lost_activation_does_not_block_other_runs(self):
+        self.fake.lose_resume = True
+        with self.assertRaises(TimeoutError): self.backend.submit(self.body)
+        run_id = next(iter(self.store.records(self.backend.settings["scope"])))["run_id"]
+        self.fake.lose_resume = False
+        self.fake.jobs.clear()
+        other = self.backend.submit({**self.body, "submission_key":str(uuid.uuid4())})["prompt_id"]
+        self.backend.refresh()
+        self.assertEqual(self.backend.jobs[run_id]["status"], "failed")
+        self.assertFalse(self.backend.jobs[run_id]["deadline"]["accepted"])
+        self.assertEqual(self.backend.jobs[other]["status"], "pending")
+        self.backend.refresh()
+        self.assertEqual(self.fake.submissions, 2)
+
+    def test_failed_job_after_lost_activation_does_not_block_other_runs(self):
+        self.fake.lose_resume = True
+        with self.assertRaises(TimeoutError): self.backend.submit(self.body)
+        job_id = next(iter(self.fake.jobs))
+        run_id = self.fake.jobs[job_id]["run_id"]
+        self.fake.lose_resume = False
+        self.fake.jobs[job_id]["Status"] = "Failed"
+        other = self.backend.submit({**self.body, "submission_key":str(uuid.uuid4())})["prompt_id"]
+        self.backend.refresh()
+        self.assertEqual(self.backend.jobs[run_id]["status"], "failed")
+        self.assertEqual(self.backend.jobs[other]["status"], "pending")
+        self.assertEqual(self.fake.submissions, 2)
+
+    def test_confirmed_removed_run_does_not_block_history_scope_change(self):
+        self.backend.submit(self.body)
+        self.fake.jobs.clear()
+        self.backend.refresh()
+        self.backend.configure({**self.backend.settings, "scope":"another-history", "enabled":False})
+        self.assertEqual(self.backend.settings["scope"], "another-history")
+
+    def test_cancelling_a_failed_attempt_pending_infrastructure_retry(self):
+        import asyncio
+        run_id = self.backend.submit(self.body)["prompt_id"]
+        job_id = next(iter(self.fake.jobs))
+        attempt = str(uuid.uuid4())
+        state = self.store.begin_attempt(run_id, attempt, job_id, "TestWorker", {"worker":"TestWorker","start":1})
+        self.store.update_attempt(run_id, attempt, state["generation"], status="failed")
+        self.backend.refresh()
+        self.assertEqual(self.backend.jobs[run_id]["status"], "pending")
+        self.assertTrue(asyncio.run(self.backend.cancel(run_id)))
+        self.assertTrue(self.backend.jobs[run_id]["deadline"]["cancel_requested"])
+        self.assertTrue(self.backend.progress[0]["cancel_requested"])
+        self.assertEqual(self.backend.progress[0]["sequence"], 1)
+
+    def test_plugin_directory_requires_a_shared_parent_directory(self):
+        with self.assertRaisesRegex(ValueError, "Custom plugin directory"):
+            validate_settings({"plugin_directory": "C:/local/plugin"})
+
+    def test_enable_cannot_persist_when_native_plugin_is_missing(self):
+        self.fake.snapshot["plugin_available"] = False
+        before = dict(self.backend.settings)
+        self.backend.settings["enabled"] = False
+        # Only health-probe filesystem writes are substituted; job/store records
+        # and any settings write still use the real temporary directory.
+        def write(path, value):
+            if Path(path) == self.backend.settings_path:
+                atomic_json(path, value)
+        with patch.object(Path, "mkdir"), patch("farm_backend.record_lock", return_value=contextlib.nullcontext()), patch("farm_backend.atomic_json", side_effect=write):
+            with self.assertRaisesRegex(ValueError, "native ComfyUI plugin"):
+                self.backend.configure(before)
+        self.assertFalse(self.backend.settings["enabled"])
+        self.assertFalse(self.backend.settings_path.exists())
+        self.assertEqual(self.fake.submissions, 0)
+
+    def test_refresh_preserves_late_acceptance_even_with_an_old_creation_time(self):
+        accepted = self.backend.submit(self.body)["prompt_id"]
+        submission = read_json(self.store.run(accepted) / "submission.json")
+        submission["created_at"] = 1
+        receipt = read_json(self.store.run(accepted) / "receipt.json")
+        self.backend.jobs, self.backend.histories, self.backend.assets = {}, {}, {}
+        def late_acceptance(scope):
+            self.backend._cache_submission(submission, receipt)
+            return []
+        with patch.object(self.store, "records", side_effect=late_acceptance):
+            self.backend.refresh()
+        self.assertIn(accepted, self.backend.jobs)
+        self.assertIn(accepted, self.backend.histories)
+        self.assertIn(accepted, self.backend.assets)
+
+    def test_acceptance_is_immediately_available_to_native_controls(self):
+        accepted = self.backend.submit(self.body)["prompt_id"]
+        self.assertEqual(self.backend.jobs[accepted]["status"], "pending")
+        self.assertEqual(self.backend.histories[accepted]["prompt"][1], accepted)
+
+    def test_missing_share_identity_retains_cached_jobs_and_prevents_new_submission(self):
+        accepted = self.backend.submit(self.body)["prompt_id"]
+        (self.store.root / "store.json").rename(self.store.root / "store.offline")
+        with self.assertRaisesRegex(RuntimeError, "storage identity"):
+            self.backend.refresh()
+        self.assertIn(accepted, self.backend.jobs)
+        with self.assertRaises(RuntimeError): self.backend.submit({**self.body, "submission_key": str(uuid.uuid4())})
+        self.assertEqual(self.fake.submissions, 1)
+
+    def test_corrupt_optional_settings_disable_native_without_breaking_node_import(self):
+        path = self.root / "bad-settings.json"
+        path.write_text("{bad json")
+        backend = FarmBackend(path, self.fake)
+        self.assertFalse(backend.settings["enabled"])
+        self.assertIn("could not be loaded", backend.health["message"])
+
+    def test_worker_process_never_starts_a_submitter_or_site_bootstrap(self):
+        with patch.dict("os.environ", {"DEADLINE_SLAVE_NAME":"Worker"}), patch("farm_backend.FarmBackend") as backend:
+            self.assertIsNone(register_backend())
+        backend.assert_not_called()
+
+    def test_result_assets_cache_and_terminal_progress_are_native_compatible(self):
+        run_id = self.backend.submit(self.body)["prompt_id"]
+        job_id = next(iter(self.fake.jobs))
+        attempt = str(uuid.uuid4())
+        state = self.store.begin_attempt(run_id, attempt, job_id, "Worker", {"worker": "Worker", "start": 1})
+        relative = "attempts/" + attempt + "/outputs/1/images/image.png"
+        image = self.store.run(run_id) / relative
+        image.parent.mkdir(parents=True); image.write_bytes(b"image")
+        atomic_json(image.parents[3] / "progress.json", {"run_id": run_id, "generation": 1, "sequence": 1,
+            "status": "in_progress", "nodes": {"1": {"value": 1, "max": 1, "prompt_id": "worker-prompt"}}})
+        result = {"schema": 1, "run_id": run_id, "attempt_id": attempt, "generation": 1,
+            "outputs": {"1": {"images": [{"filename": "image.png", "subfolder": "", "type": "output"}]}},
+            "files": [{"path": relative, "size": 5, "sha256": hashlib.sha256(b"image").hexdigest(), "node_id": "1", "output_key": "images", "output_index": 0}]}
+        self.store.commit(run_id, attempt, 1, result)
+        self.fake.jobs[job_id]["Status"] = "Completed"
+        self.backend.output_root = self.root / "local-output"
+        self.backend.refresh()
+        asset = self.backend.assets[run_id][0]
+        self.assertEqual(asset["name"], "image.png")
+        self.assertEqual(str(uuid.UUID(asset["id"])), asset["id"])
+        self.assertIn("deadline-native", asset["preview_url"])
+        self.assertEqual((self.backend.output_root / asset["subfolder"] / asset["filename"]).read_bytes(), b"image")
+        event = self.backend.progress[0]
+        self.assertEqual(event["sequence"], 6)
+        self.assertEqual(event["status"], "completed")
+        self.assertEqual(event["nodes"]["1"]["prompt_id"], run_id)
+
+    def test_native_output_loader_is_staged_and_retry_survives_original_cache_loss(self):
+        output_root = self.root / "artist-output"
+        image = output_root / "deadline-native" / "source" / "image.png"
+        image.parent.mkdir(parents=True); image.write_bytes(b"image")
+        with patch.dict("sys.modules", {"folder_paths": types.SimpleNamespace(
+                get_input_directory=lambda: str(self.root / "input"), get_output_directory=lambda: str(output_root))}):
+            body = copy.deepcopy(self.body)
+            body["prompt"]["2"] = {"class_type": "LoadImage", "inputs": {"image": "deadline-native/source/image.png [output]"}}
+            original = self.backend.submit(body)["prompt_id"]
+            self.fake.jobs[next(iter(self.fake.jobs))]["Status"] = "Failed"
+            self.backend.refresh()
+            image.unlink()
+            retried = self.backend.retry(original, str(uuid.uuid4()))["prompt_id"]
+            original_prompt = read_json(self.store.run(original) / "prompt.json")
+            retry_prompt = read_json(self.store.run(retried) / "prompt.json")
+            self.assertEqual(original_prompt, retry_prompt)
+            self.assertNotIn("/", retry_prompt["2"]["inputs"]["image"])
+            self.assertLess(len(retry_prompt["2"]["inputs"]["image"]), 40)
+            self.assertTrue((self.store.run(retried) / "inputs" / retry_prompt["2"]["inputs"]["image"]).is_file())
+
+
+if __name__ == "__main__": unittest.main()

tests/test_farm_discovery.py

@@ -0,0 +1,66 @@
+import json
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import Mock
+
+from farm_backend import FarmBackend
+
+
+class DiscoveryTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.command = Mock(return_value="COMFY_NATIVE_FARM=" + json.dumps({"workers": [{"name": "M28"}], "pools": ["test"], "groups": ["none"]}))
+        self.backend = FarmBackend(Path(self.temp.name) / "settings.json", self.command)
+
+    def test_cache_is_read_only_and_copy_isolated(self):
+        first = self.backend.discover()
+        first["workers"].clear()
+        self.assertEqual(self.backend.discover()["workers"], [{"name": "M28"}])
+        self.assertEqual(self.command.call_count, 2)
+        self.assertEqual(self.command.call_args_list[0].args[0], "-ExecuteScript")
+        self.assertEqual(self.command.call_args_list[1].args[0], "-GetCurrentUserName")
+
+    def test_failed_refresh_retains_last_good_workers(self):
+        good = self.backend.discover()
+        self.command.side_effect = TimeoutError("repository unavailable")
+        failed = self.backend.discover(True)
+        self.assertFalse(failed["connected"])
+        self.assertEqual(failed["workers"], good["workers"])
+        self.assertEqual(failed["last_refresh"], good["last_refresh"])
+        self.assertIn("unavailable", failed["message"])
+
+    def test_malformed_snapshot_is_a_diagnostic(self):
+        self.command.return_value = 'COMFY_NATIVE_FARM={"workers":null}'
+        result = self.backend.discover()
+        self.assertFalse(result["connected"])
+        self.assertIn("Invalid", result["message"])
+
+    def test_plugin_directory_is_part_of_discovery_cache_identity(self):
+        self.backend.discover(plugin_directory="\\\\farm\\first\\plugins")
+        self.backend.discover(plugin_directory="\\\\farm\\first\\plugins")
+        self.assertEqual(self.command.call_count, 2)
+        self.backend.discover(plugin_directory="\\\\farm\\second\\plugins")
+        self.assertEqual(self.command.call_count, 4)
+        self.assertEqual(self.command.call_args_list[-2].args[-1], "\\\\farm\\second\\plugins")
+
+    def test_failed_plugin_path_change_cannot_reuse_old_plugin_availability(self):
+        self.command.return_value = 'COMFY_NATIVE_FARM={"workers":[],"pools":[],"groups":[],"plugin_available":true}'
+        self.backend.discover(plugin_directory="\\\\farm\\first\\plugins")
+        self.command.side_effect = TimeoutError("unavailable")
+        failed = self.backend.discover(plugin_directory="\\\\farm\\second\\plugins")
+        self.assertFalse(failed["connected"])
+        self.assertFalse(failed["plugin_available"])
+
+    def test_optimistic_settings_conflict_cannot_overwrite(self):
+        original = dict(self.backend.settings)
+        self.backend.configure({**original, "priority": 60})
+        with self.assertRaisesRegex(ValueError, "changed elsewhere"):
+            self.backend.configure({**original, "priority": 90}, expected=original)
+        self.assertEqual(self.backend.settings["priority"], 60)
+
+    def test_matching_snapshot_can_apply(self):
+        original = dict(self.backend.settings)
+        self.backend.configure({**original, "priority": 90}, expected=original)
+        self.assertEqual(self.backend.settings["priority"], 90)

tests/test_farm_protocol.py

@@ -0,0 +1,135 @@
+import hashlib
+import json
+import os
+import tempfile
+import threading
+import unittest
+import uuid
+from pathlib import Path
+
+from farm_protocol import RunStore, atomic_json, read_json, safe_path, _read_unc
+
+
+class StoreTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.store = RunStore(self.temp.name)
+        self.run_id, self.attempt = str(uuid.uuid4()), str(uuid.uuid4())
+        directory = self.store.run(self.run_id)
+        atomic_json(directory / "receipt.json", {"job_id": "job"})
+        atomic_json(directory / "control.json", {"generation": 0, "status": "pending", "cancel_requested": False, "result": None})
+
+    def begin(self, attempt=None, start=1):
+        return self.store.begin_attempt(self.run_id, attempt or self.attempt, "job", "Worker", {"worker": "Worker", "start": start})
+
+    @unittest.skipUnless(os.name == "nt", "Windows record reads")
+    def test_unc_reader_handles_bom_exact_buffer_boundary_and_large_records(self):
+        path = Path(self.temp.name) / "record.json"
+        for size in (83, 65536, 131073):
+            value = {"data":"a" * (size - 15)}
+            data = b"\xef\xbb\xbf" + json.dumps(value).encode("utf-8")
+            self.assertEqual(len(data), size)
+            path.write_bytes(data)
+            self.assertEqual(json.loads(_read_unc(path)), value)
+        path.unlink()
+        with self.assertRaises(FileNotFoundError): _read_unc(path)
+
+    def test_execution_mode_is_part_of_the_atomic_attempt_claim(self):
+        state = self.store.begin_attempt(self.run_id, self.attempt, "job", "Worker",
+            {"worker": "Worker", "start": 1}, execution_mode="task-owned")
+        self.assertEqual(self.store.control(self.run_id)["execution_mode"], "task-owned")
+        self.assertEqual(state["generation"], 1)
+        replacement = self.store.begin_attempt(self.run_id, str(uuid.uuid4()), "job", "Worker",
+            {"worker": "Worker", "start": 2}, execution_mode="persistent")
+        self.assertEqual(replacement["execution_mode"], "persistent")
+        self.assertEqual(self.store.control(self.run_id)["generation"], 2)
+
+    def test_late_startup_failure_cannot_overwrite_confirmed_cancellation(self):
+        self.begin()
+        self.store.cancel(self.run_id)
+        self.store.update_attempt(self.run_id, self.attempt, 1, status="cancelled")
+        self.assertFalse(self.store.update_attempt(self.run_id, self.attempt, 1, status="failed"))
+        self.assertEqual(self.store.control(self.run_id)["status"], "cancelled")
+
+    def manifest(self, attempt=None, generation=1):
+        attempt = attempt or self.attempt
+        relative = "attempts/" + attempt + "/outputs/image.png"
+        path = safe_path(self.store.run(self.run_id), relative)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(b"validated output")
+        return {"schema": 1, "run_id": self.run_id, "attempt_id": attempt, "generation": generation,
+            "files": [{"path": relative, "size": path.stat().st_size, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}]}
+
+    def test_new_assignment_fences_old_writer_and_stale_assignment(self):
+        self.begin()
+        old = self.manifest()
+        new = str(uuid.uuid4())
+        state = self.begin(new, 2)
+        self.assertEqual(state["generation"], 2)
+        self.assertFalse(self.store.commit(self.run_id, self.attempt, 1, old))
+        with self.assertRaisesRegex(ValueError, "Stale"):
+            self.begin(str(uuid.uuid4()), 1)
+        self.assertTrue(self.store.commit(self.run_id, new, 2, self.manifest(new, 2)))
+
+    def test_duplicate_assignment_and_callback_cannot_claim_again(self):
+        self.begin()
+        with self.assertRaisesRegex(ValueError, "already has"):
+            self.begin(str(uuid.uuid4()))
+        result = self.manifest()
+        self.assertTrue(self.store.commit(self.run_id, self.attempt, 1, result))
+        self.assertTrue(self.store.commit(self.run_id, self.attempt, 1, result))
+        self.assertIsNone(self.begin(str(uuid.uuid4()), 2))
+        self.assertEqual(self.store.control(self.run_id)["generation"], 1)
+
+    def test_cancel_and_commit_have_one_deterministic_winner(self):
+        self.begin()
+        manifest = self.manifest()
+        barrier = threading.Barrier(3)
+        results = {}
+        def cancel():
+            barrier.wait()
+            results["cancel"] = self.store.cancel(self.run_id)
+        def commit():
+            barrier.wait()
+            results["commit"] = self.store.commit(self.run_id, self.attempt, 1, manifest)
+        threads = [threading.Thread(target=cancel), threading.Thread(target=commit)]
+        for thread in threads: thread.start()
+        barrier.wait()
+        for thread in threads: thread.join()
+        self.assertNotEqual(results["cancel"], results["commit"])
+        state = self.store.control(self.run_id)
+        self.assertEqual(bool(state["result"]), results["commit"])
+        self.assertEqual(state["cancel_requested"], results["cancel"])
+
+    def test_commit_acknowledgement_loss_survives_store_restart(self):
+        self.begin()
+        manifest = self.manifest()
+        self.store.commit(self.run_id, self.attempt, 1, manifest)
+        replacement = RunStore(self.temp.name)
+        self.assertEqual(replacement.result(self.run_id), manifest)
+        self.assertIsNone(replacement.begin_attempt(self.run_id, str(uuid.uuid4()), "job", "Worker", {"start": 2, "worker": "Worker"}))
+
+    def test_invalid_missing_tampered_or_escaping_outputs_cannot_commit(self):
+        self.begin()
+        manifest = self.manifest()
+        manifest["files"][0]["path"] = "../outside.png"
+        with self.assertRaises(ValueError): self.store.commit(self.run_id, self.attempt, 1, manifest)
+        manifest = self.manifest()
+        safe_path(self.store.run(self.run_id), manifest["files"][0]["path"]).write_bytes(b"corrupt")
+        with self.assertRaises(ValueError): self.store.commit(self.run_id, self.attempt, 1, manifest)
+        self.assertIsNone(self.store.control(self.run_id)["result"])
+
+    def test_wrong_deadline_mapping_and_path_traversal_rejected(self):
+        with self.assertRaises(ValueError): self.store.begin_attempt(self.run_id, self.attempt, "other", "Worker", {"start": 1})
+        for value in ("../outside", "C:/outside", "/outside", "\\\\host\\share\\file", "image.png:stream"):
+            with self.assertRaises(ValueError): safe_path(self.temp.name, value)
+
+    def test_reassigned_deadline_task_cannot_publish_before_new_generation_claim(self):
+        self.begin()
+        result = self.manifest()
+        self.assertFalse(self.store.commit(self.run_id, self.attempt, 1, result, lambda: False))
+        self.assertIsNone(self.store.control(self.run_id)["result"])
+
+
+if __name__ == "__main__": unittest.main()

tests/test_farm_readiness.py

@@ -0,0 +1,97 @@
+import copy
+import unittest
+
+from farm_readiness import readiness
+
+
+class ReadinessTests(unittest.TestCase):
+    def setUp(self):
+        self.settings = {"enabled": True, "root": "\\\\farm\\native", "scope": "artist", "workers": "GPU-Worker", "pool": "gpu", "group": "comfy"}
+        self.snapshot = {"connected": True, "message": "Repository connected", "plugin_available": True,
+            "storage": {"connected": True, "message": "Storage available"}, "workers": [{"name": "GPU-Worker", "enabled": True,
+                "status": "Idle", "session_configured": True, "pools": ["gpu"], "groups": ["comfy"]}]}
+
+    def test_eligible_worker_status_does_not_change_deadline_scheduling(self):
+        for status in ("Idle", "Rendering", "Offline"):
+            self.snapshot["workers"][0]["status"] = status
+            self.assertTrue(readiness(self.settings, self.snapshot)["ready"])
+
+    def test_names_and_pool_group_membership_match_case_insensitively(self):
+        self.settings.update(workers="gpu-worker,GPU-WORKER", pool="GPU", group="COMFY")
+        result = readiness(self.settings, self.snapshot)
+        self.assertTrue(result["ready"])
+        self.assertEqual(len(result["workers"]), 1)
+        self.assertEqual(result["workers"][0]["name"], "GPU-Worker")
+
+    def test_invalid_worker_selection_has_actionable_reasons(self):
+        self.settings["workers"] = "GPU-Worker,Missing"
+        self.snapshot["workers"][0].update(enabled=False, session_configured=False, pools=[], groups=[])
+        result = readiness(self.settings, self.snapshot)
+        self.assertFalse(result["ready"])
+        self.assertEqual(result["code"], "workers")
+        self.assertIn("outside selected group", result["message"])
+        self.assertIn("Missing: not found", result["message"])
+
+    def test_storage_repository_and_plugin_failures_are_distinguished(self):
+        for field, code in (("storage", "storage"), ("connected", "repository"), ("plugin_available", "plugin")):
+            snapshot = copy.deepcopy(self.snapshot)
+            snapshot[field] = {"connected": False, "message": "Storage unavailable"} if field == "storage" else False
+            self.assertEqual(readiness(self.settings, snapshot)["code"], code)
+
+    def test_local_storage_is_rejected_even_when_health_claims_connected(self):
+        self.settings["root"] = "C:/local"
+        self.assertEqual(readiness(self.settings, self.snapshot)["code"], "storage")
+
+    def test_global_configuration_error_blocks_even_a_registered_worker(self):
+        self.snapshot["configuration_error"] = "Malformed NativeWorkerSessions JSON"
+        result = readiness(self.settings, self.snapshot)
+        self.assertFalse(result["ready"])
+        self.assertEqual(result["code"], "session_configuration")
+
+    def test_empty_allowlist_uses_deadline_scheduling_and_admin_disable_blocks(self):
+        self.settings["workers"] = ", ,"
+        self.assertTrue(readiness(self.settings, self.snapshot)["ready"])
+        self.settings["enabled"] = False
+        self.assertEqual(readiness(self.settings, self.snapshot)["code"], "enabled")
+
+    def test_existing_workers_do_not_require_session_owners(self):
+        self.snapshot["workers"][0]["session_configured"] = False
+        self.assertTrue(readiness(self.settings, self.snapshot)["ready"])
+
+    def test_deadline_user_is_independent_of_history_scope(self):
+        self.settings["scope"] = "legacy-history"
+        self.snapshot["deadline_user"] = "artist"
+        self.snapshot["workers"][0].update(scheduling_mode="UserJobs", user_job_names=["artist"])
+        self.assertTrue(readiness(self.settings, self.snapshot)["ready"])
+
+    def test_user_scope_changes_cannot_create_ready_jobs_the_worker_will_never_take(self):
+        worker = self.snapshot["workers"][0]
+        worker.update(scheduling_mode="UserJobs", user_job_names=["Artist-comfy"])
+        self.settings["scope"] = "ARTIST-COMFY"
+        self.assertTrue(readiness(self.settings, self.snapshot)["ready"])
+        self.settings["scope"] = "artist"
+        result = readiness(self.settings, self.snapshot)
+        self.assertFalse(result["ready"])
+        self.assertEqual(result["code"], "workers")
+        self.assertIn("does not accept Deadline user artist", result["message"])
+        self.assertIn("Artist-comfy", result["message"])
+
+    def test_empty_user_allowlist_and_unverified_scheduling_modes_block_submission(self):
+        worker = self.snapshot["workers"][0]
+        worker.update(scheduling_mode="UserJobs", user_job_names=[])
+        self.assertFalse(readiness(self.settings, self.snapshot)["ready"])
+        self.assertIn("no users", readiness(self.settings, self.snapshot)["message"])
+        for mode in ("ListedJobs", "Unknown"):
+            worker["scheduling_mode"] = mode
+            result = readiness(self.settings, self.snapshot)
+            self.assertFalse(result["ready"])
+            self.assertIn("could not be verified", result["message"])
+
+    def test_machine_jobs_require_the_verified_machine_user(self):
+        worker = self.snapshot["workers"][0]
+        worker.update(scheduling_mode="MachineJobs", machine_user="ARTIST")
+        self.assertTrue(readiness(self.settings, self.snapshot)["ready"])
+        self.settings["scope"] = "artist-comfy"
+        self.assertFalse(readiness(self.settings, self.snapshot)["ready"])
+        worker["machine_user"] = ""
+        self.assertIn("could not be verified", readiness(self.settings, self.snapshot)["message"])

tests/test_farm_session_owner.py

@@ -0,0 +1,112 @@
+import argparse
+import subprocess
+import tempfile
+import time
+import unittest
+import urllib.error
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+from farm_protocol import read_json
+from farm_session_owner import SessionOwner
+
+
+class SessionOwnershipTests(unittest.TestCase):
+    def setUp(self):
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        root = Path(temporary.name)
+        self.args = argparse.Namespace(state_root=root, comfy_root=root, worker="Worker", gpu_uuid="GPU-test",
+            borrowed_url="", borrowed_input_root=None, borrowed_output_root=None, borrowed_temp_root=None,
+            comfy_port=8198, broker_port=8202, exclusive_worker=True, idle_seconds=1)
+        self.owner = SessionOwner(self.args)
+        self.owner.owned = Mock()
+        self.owner.owned.process.poll.return_value = None
+        self.owner.reservation = {"token": "token", "prompt_id": "own-prompt", "job_id": "job",
+            "reserved_at": time.time(), "last_client_at": time.monotonic() - 30}
+        self.owner.status = Mock(return_value={"state": "running"})
+        self.owner.check_identity = Mock()
+
+    def test_missing_worker_plugin_reports_404_immediately_and_stops_owned_child(self):
+        self.owner.owned = None
+        self.owner.reservation = None
+        self.args.custom_nodes = []
+        self.args.extra_args = []
+        process = Mock()
+        process.process.poll.return_value = None
+        responses = [urllib.error.URLError("refused"),
+            urllib.error.HTTPError(self.owner.url + "/deadline/session", 404, "Missing", {}, None)]
+        with patch("farm_session_owner.comfy_request", side_effect=responses), \
+                patch("farm_session_owner.subprocess.check_output", return_value="0, GPU-test\n"), \
+                patch("farm_session_owner.OwnedProcess", return_value=process):
+            with self.assertRaisesRegex(RuntimeError, "lacks ComfyUI-Deadline-Plugin"):
+                self.owner.reserve({"worker":"Worker","gpu_uuid":"GPU-test"})
+        process.close.assert_called_once()
+        self.assertIsNone(self.owner.owned)
+
+    def test_stale_heartbeat_cannot_stop_current_assignment(self):
+        self.owner.assignment_active = Mock(return_value=True)
+        with patch("farm_session_owner.comfy_request") as request:
+            self.owner.idle_tick()
+        request.assert_not_called()
+        self.owner.owned.close.assert_not_called()
+        self.assertIsNotNone(self.owner.reservation)
+
+    def test_deadline_timeout_preserves_active_ownership(self):
+        with patch("farm_session_owner.subprocess.run", side_effect=subprocess.TimeoutExpired("deadline", 10)), patch("farm_session_owner.comfy_request") as request:
+            self.owner.idle_tick()
+        request.assert_not_called()
+        self.owner.owned.close.assert_not_called()
+
+    def test_lost_assignment_interrupts_only_reserved_prompt(self):
+        self.owner.assignment_active = Mock(return_value=False)
+        with patch("farm_session_owner.comfy_request") as request:
+            self.owner.idle_tick()
+        request.assert_called_once_with(self.owner.url + "/api/jobs/own-prompt/cancel", {})
+        self.owner.owned.close.assert_not_called()
+
+    def test_finished_prompt_remains_reserved_while_task_publishes(self):
+        self.owner.status.return_value = {"state": "finished"}
+        self.owner.assignment_active = Mock(return_value=True)
+        self.owner.idle_tick()
+        self.assertIsNotNone(self.owner.reservation)
+        self.owner.owned.close.assert_not_called()
+
+    def test_shared_worker_releases_gpu_after_abandoned_finished_task(self):
+        self.args.exclusive_worker = False
+        self.owner.status.return_value = {"state": "finished"}
+        self.owner.assignment_active = Mock(return_value=False)
+        child = self.owner.owned
+        self.owner.idle_tick()
+        child.close.assert_called_once()
+        self.assertIsNone(self.owner.reservation)
+        self.assertIsNone(read_json(self.args.state_root / "reservation.json")["reservation"])
+
+    def test_borrowed_idle_endpoint_is_never_terminated(self):
+        self.owner.borrowed, self.owner.owned, self.owner.reservation = True, None, None
+        self.owner.last_active = time.monotonic() - 10
+        self.owner.stop = Mock()
+        with patch("farm_session_owner.comfy_request", return_value={"queue_running": [], "queue_pending": []}):
+            self.owner.idle_tick()
+        self.owner.stop.assert_not_called()
+
+    def test_idle_owned_endpoint_checks_queue_before_shutdown(self):
+        self.owner.reservation = None
+        self.owner.last_active = time.monotonic() - 10
+        child = self.owner.owned
+        with patch("farm_session_owner.comfy_request", return_value={"queue_running": ["artist"], "queue_pending": []}):
+            self.owner.idle_tick()
+        child.close.assert_not_called()
+        with patch("farm_session_owner.comfy_request", return_value={"queue_running": [], "queue_pending": []}):
+            self.owner.idle_tick()
+        child.close.assert_called_once()
+
+    def test_borrowed_off_host_endpoint_is_rejected(self):
+        self.args.borrowed_url = "http://other-host:8188"
+        with self.assertRaisesRegex(ValueError, "worker-local"):
+            SessionOwner(self.args)
+
+    def test_dead_child_clears_durable_reservation(self):
+        self.owner.owned.process.poll.return_value = 1
+        self.owner.idle_tick()
+        self.assertIsNone(read_json(self.args.state_root / "reservation.json")["reservation"])

tests/test_native_jobs_adapter.py

@@ -0,0 +1,152 @@
+import copy
+import json
+import unittest
+import uuid
+
+from aiohttp import web
+from aiohttp.test_utils import TestClient, TestServer
+
+from native_jobs_adapter import NativeJobsAdapter, select_jobs
+from scripts.native_queue_spike import FixtureProvider, snapshot
+
+
+def job(status="pending", created=1):
+    return {"id": str(uuid.uuid4()), "status": status, "create_time": created,
+            "outputs_count": 0, "previewable_outputs_count": 0, "workflow_id": "local"}
+
+
+class NativeJobsTests(unittest.IsolatedAsyncioTestCase):
+    async def asyncSetUp(self):
+        self.provider = FixtureProvider()
+        self.farm_id = self.provider.submit({**snapshot(), "submission_key": str(uuid.uuid4())})["prompt_id"]
+        self.local = [job(created=10), job("completed", 20)]
+        self.local_requests = []
+        self.local_cancel_ids = []
+        app = web.Application(middlewares=[NativeJobsAdapter(self.provider).middleware])
+
+        async def local(request):
+            self.local_requests.append((request.method, request.path, dict(request.query)))
+            if request.method == "GET" and request.path.endswith("/jobs"):
+                return web.json_response(select_jobs(self.local, request.query))
+            if request.path.endswith("/jobs/cancel"):
+                ids = (await request.json())["job_ids"]
+                self.local_cancel_ids += [j["id"] for j in self.local if j["id"] in ids and j["status"] == "pending"]
+                return web.json_response({"cancelled": bool(self.local_cancel_ids)})
+            if request.path.endswith("/queue") and request.method == "GET":
+                return web.json_response({"queue_running": [], "queue_pending": []})
+            if request.method == "GET" and request.path.endswith("/history"):
+                return web.json_response({})
+            if request.method == "GET":
+                return web.json_response({"local": True})
+            return web.Response(status=200)
+
+        app.router.add_route("*", "/{path:.*}", local)
+        self.client = TestClient(TestServer(app))
+        await self.client.start_server()
+
+    async def asyncTearDown(self):
+        await self.client.close()
+
+    async def test_combined_pagination_and_filters_before_slicing(self):
+        response = await self.client.get("/api/jobs?status=pending&limit=1&offset=1&sort_order=asc")
+        data = await response.json()
+        self.assertEqual(data["pagination"], {"offset": 1, "limit": 1, "total": 2, "has_more": False})
+        self.assertEqual(data["jobs"][0]["id"], self.farm_id)
+        self.assertEqual(self.local_requests[0][2], {})
+        response = await self.client.get("/api/jobs?workflow_id=local")
+        self.assertEqual((await response.json())["pagination"]["total"], 2)
+
+    async def test_over_200_active_runs_have_correct_total_and_next_page(self):
+        for _ in range(205):
+            self.provider.submit({**snapshot(), "submission_key": str(uuid.uuid4())})
+        first = await (await self.client.get("/api/jobs?status=pending,in_progress&limit=200&offset=0")).json()
+        second = await (await self.client.get("/api/jobs?status=pending,in_progress&limit=200&offset=200")).json()
+        self.assertEqual(first["pagination"]["total"], 207)
+        self.assertTrue(first["pagination"]["has_more"])
+        self.assertEqual(len(second["jobs"]), 7)
+        self.assertFalse(set(j["id"] for j in first["jobs"]) & set(j["id"] for j in second["jobs"]))
+
+    async def test_local_detail_and_unrelated_routes_pass_through(self):
+        for path in ("/api/jobs/" + self.local[0]["id"], "/system_stats", "/object_info"):
+            self.assertEqual(await (await self.client.get(path)).json(), {"local": True})
+
+    async def test_owner_idle_query_excludes_remote_execution_records(self):
+        visible = await (await self.client.get("/api/queue")).json()
+        self.assertEqual(len(visible["queue_pending"]), 1)
+        actual = await (await self.client.get("/api/queue?deadline_local=1")).json()
+        self.assertEqual(actual, {"queue_pending": [], "queue_running": []})
+
+    async def test_farm_workflow_detail_assets_and_legacy_history(self):
+        self.provider.transition(self.farm_id, "completed")
+        data = await (await self.client.get("/api/jobs/" + self.farm_id)).json()
+        self.assertEqual(data["workflow"]["prompt"]["1"]["class_type"], "EmptyImage")
+        self.assertEqual(data["outputs_count"], 1)
+        self.assertIn("workflow", data["workflow"]["extra_data"]["extra_pnginfo"])
+        assets = await (await self.client.get(f"/api/jobs/{self.farm_id}/assets?limit=1")).json()
+        self.assertEqual(assets["assets"][0]["node_id"], "2")
+        self.assertEqual(assets["pagination"]["total"], 1)
+        history = await (await self.client.get("/api/history/" + self.farm_id)).json()
+        self.assertEqual(history[self.farm_id]["prompt"][1], self.farm_id)
+
+    async def test_mixed_cancel_keeps_local_handler_and_is_idempotent(self):
+        ids = [self.local[0]["id"], self.farm_id]
+        data = await (await self.client.post("/api/jobs/cancel", json={"job_ids": ids})).json()
+        self.assertTrue(data["cancelled"])
+        self.assertEqual(self.local_cancel_ids, [self.local[0]["id"]])
+        self.assertEqual(self.provider.records[self.farm_id]["status"], "cancelled")
+        data = await (await self.client.post(f"/api/jobs/{self.farm_id}/cancel")).json()
+        self.assertFalse(data["cancelled"])
+
+    async def test_invalid_mixed_cancel_has_no_side_effects(self):
+        for invalid in ({}, "uppercase", self.farm_id.upper(), None):
+            response = await self.client.post("/api/jobs/cancel", json={"job_ids": [self.farm_id, invalid]})
+            self.assertEqual(response.status, 400)
+        self.assertEqual(self.provider.records[self.farm_id]["status"], "pending")
+        self.assertEqual(self.local_cancel_ids, [])
+
+    async def test_targeted_interrupt_and_queue_clear_only_address_farm_jobs(self):
+        self.provider.transition(self.farm_id, "in_progress", 50)
+        self.assertEqual((await self.client.post("/api/interrupt", json={"prompt_id": self.farm_id})).status, 200)
+        self.assertEqual(self.provider.records[self.farm_id]["status"], "cancelled")
+        pending = self.provider.submit({**snapshot(), "submission_key": str(uuid.uuid4())})["prompt_id"]
+        self.assertEqual((await self.client.post("/api/queue", json={"clear": True})).status, 200)
+        self.assertEqual(self.provider.records[pending]["status"], "cancelled")
+
+    async def test_history_delete_hides_record_without_destroying_snapshot(self):
+        self.provider.transition(self.farm_id, "failed")
+        await self.client.post("/api/history", json={"delete": [self.farm_id]})
+        self.assertIn(self.farm_id, self.provider.records)
+        self.assertIsNone(await self.provider.get_job(self.farm_id))
+        self.assertEqual((await (await self.client.get("/api/jobs")).json())["pagination"]["total"], 2)
+
+    async def test_invalid_query_never_runs_local_handler(self):
+        for query in ("status=bad", "limit=0", "offset=NaN", "sort_by=bogus", "sort_order=nope"):
+            self.assertEqual((await self.client.get("/api/jobs?" + query)).status, 400)
+        self.assertEqual(self.local_requests, [])
+
+    async def test_lost_frontend_wrapper_cannot_execute_farm_snapshot_locally(self):
+        payload = snapshot()
+        payload["extra_data"]["extra_pnginfo"]["workflow"]["__deadline_native_queue"] = str(uuid.uuid4())
+        response = await self.client.post("/api/prompt", json=payload)
+        self.assertEqual(response.status, 409)
+        self.assertEqual(self.local_requests, [])
+
+
+class FixtureSnapshotTests(unittest.TestCase):
+    def test_transport_retries_and_different_payload_conflict(self):
+        provider = FixtureProvider()
+        body = {**snapshot(), "submission_key": str(uuid.uuid4())}
+        accepted = provider.submit(body)
+        self.assertEqual(provider.submit(copy.deepcopy(body)), accepted)
+        changed = copy.deepcopy(body)
+        changed["prompt"]["1"]["inputs"]["color"] = 999
+        with self.assertRaisesRegex(ValueError, "different snapshot"):
+            provider.submit(changed)
+        self.assertEqual(len(provider.records), 1)
+
+    def test_browser_credentials_are_not_retained(self):
+        provider = FixtureProvider()
+        body = {**snapshot(), "submission_key": str(uuid.uuid4())}
+        body["extra_data"].update(auth_token_comfy_org="private_token", api_key_comfy_org="private_key")
+        run_id = provider.submit(body)["prompt_id"]
+        self.assertNotIn("private_", json.dumps(provider.records[run_id]))

tests/test_native_queue_task.py

@@ -0,0 +1,128 @@
+import importlib.util
+import json
+import subprocess
+import tempfile
+import unittest
+import uuid
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+from farm_protocol import RunStore, atomic_json
+
+specification = importlib.util.spec_from_file_location("native_task_under_test", Path(__file__).parents[1] / "plugins/ComfyUI/native_queue_task.py")
+task = importlib.util.module_from_spec(specification)
+specification.loader.exec_module(task)
+
+
+class NativeTaskFailureTests(unittest.TestCase):
+    def setUp(self):
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        root = Path(temporary.name)
+        self.store = RunStore(root / "share")
+        self.run_id, self.attempt = str(uuid.uuid4()), str(uuid.uuid4())
+        atomic_json(self.store.run(self.run_id) / "receipt.json", {"job_id": "job"})
+        atomic_json(self.store.run(self.run_id) / "control.json", {"generation": 0, "status": "pending", "cancel_requested": False, "result": None})
+        state = self.store.begin_attempt(self.run_id, self.attempt, "job", "Worker", {"worker": "Worker", "start": 1})
+        self.spec = {"root": str(self.store.root), "run_id": self.run_id, "attempt_id": self.attempt,
+            "job_id": "job", "generation": state["generation"], "skip": False, "token": "reservation",
+            "broker": "http://127.0.0.1:8202", "timeout_seconds": 0, "payload": {"prompt": {}},
+            "identity": {"output_root": str(root / "outputs"), "temp_root": str(root / "temp"), "session_id": "session"},
+            "expected_outputs": ["1"], "assignment": {"worker": "Worker", "start": 639269075466550000}}
+
+    def history(self, filename):
+        return {"outputs": {"1": {"images": [{"filename": filename, "type": "output"}]}}, "status": {"status_str": "success"}}
+
+    def test_missing_output_cannot_publish_success(self):
+        with self.assertRaisesRegex(ValueError, "Missing ComfyUI output"):
+            task.publish_outputs(self.store, self.spec, self.history("missing.png"))
+        self.assertIsNone(self.store.control(self.run_id)["result"])
+
+    def test_no_file_from_required_output_node_fails(self):
+        with self.assertRaisesRegex(ValueError, "no validated files"):
+            task.publish_outputs(self.store, self.spec, {"outputs": {"1": {"text": ["hello"]}}, "status": {"status_str": "success"}})
+
+    def test_output_traversal_rejected(self):
+        with self.assertRaisesRegex(ValueError, "escapes"):
+            task.publish_outputs(self.store, self.spec, self.history("../artist.png"))
+
+    def test_timeout_cancels_only_the_reserved_prompt(self):
+        def broker(url, body):
+            return {"prompt_id": "own-prompt"} if url.endswith("/queue") else {}
+        with patch.object(task, "request", side_effect=broker) as request:
+            with self.assertRaises(TimeoutError): task.run(self.spec)
+        self.assertTrue(any(call.args == (self.spec["broker"] + "/cancel", {"token": "reservation"}) for call in request.call_args_list))
+        self.assertTrue(all(call.args[0].startswith(self.spec["broker"]) for call in request.call_args_list))
+        self.assertEqual(self.store.control(self.run_id)["status"], "failed")
+        self.assertIsNone(self.store.control(self.run_id)["result"])
+
+    def test_assignment_requires_current_worker_status_and_start(self):
+        fields = "COMFY_NATIVE_ASSIGNMENT=" + json.dumps({"status": "Rendering", "worker": "Worker", "start": 639269075466550000})
+        with patch.object(task.subprocess, "run", return_value=Mock(returncode=0, stdout=fields)):
+            self.assertTrue(task.assignment_is_current(self.spec))
+        for stale in (fields.replace("Worker", "Other"), fields.replace("Rendering", "Queued"), fields.replace("639269075466550000", "639269075466550001")):
+            with patch.object(task.subprocess, "run", return_value=Mock(returncode=0, stdout=stale)):
+                self.assertFalse(task.assignment_is_current(self.spec))
+
+    def test_unknown_assignment_never_authorizes_publication(self):
+        with patch.object(task.subprocess, "run", side_effect=subprocess.TimeoutExpired("deadline", 30)):
+            with self.assertRaises(subprocess.TimeoutExpired): task.assignment_is_current(self.spec)
+
+
+class TaskOwnershipTests(unittest.TestCase):
+    def test_task_owner_resolves_deadline_affinity_and_verifies_its_pid(self):
+        with tempfile.TemporaryDirectory() as directory:
+            plugin = Mock()
+            plugin.temp_dir = directory
+            plugin._resolve_comfyui_install_path.return_value = directory
+            plugin._get_gpu_from_worker_affinity.return_value = "1"
+            plugin.GetSlaveName.return_value = "ExistingWorker"
+            owner = Mock()
+            owner.process.pid = 1234
+            owner.process.poll.return_value = None
+            with patch("farm_session_owner.OwnedProcess", return_value=owner) as launch, \
+                    patch.object(task.subprocess, "check_output", return_value="0, GPU-zero\n1, GPU-one\n"), \
+                    patch.object(task, "request", return_value={"owner_pid":1234}):
+                config = task.start_task_owner(plugin, startup_seconds=600)
+            command = launch.call_args.args[0]
+            plugin._set_deadline_environment_variables.assert_called_once()
+            self.assertEqual(command[command.index("--gpu-uuid") + 1], "GPU-one")
+            self.assertNotIn("--exclusive-worker", command)
+            self.assertEqual(command[command.index("--startup-seconds") + 1], "600")
+            self.assertEqual(config["gpu_uuid"], "GPU-one")
+            plugin.native_owner_log.close()
+
+    def test_cleanup_closes_task_owner_even_when_reservation_cleanup_fails(self):
+        plugin = Mock(native_owner=Mock(), native_owner_log=Mock())
+        owner, log = plugin.native_owner, plugin.native_owner_log
+        with patch.object(task, "cleanup_reservation", side_effect=RuntimeError("broker unavailable")):
+            with self.assertRaises(RuntimeError):
+                task.cleanup(plugin)
+        owner.close.assert_called_once()
+        log.close.assert_called_once()
+        self.assertIsNone(plugin.native_owner)
+
+    def test_task_owner_does_not_use_another_process_at_its_port(self):
+        with tempfile.TemporaryDirectory() as directory:
+            plugin = Mock(temp_dir=directory)
+            plugin._resolve_comfyui_install_path.return_value = directory
+            plugin._get_gpu_from_worker_affinity.return_value = "0"
+            owner = Mock()
+            owner.process.pid = 1234
+            owner.process.poll.return_value = None
+            with patch("farm_session_owner.OwnedProcess", return_value=owner), \
+                    patch.object(task.subprocess, "check_output", return_value="0, GPU-zero\n"), \
+                    patch.object(task, "request", return_value={"owner_pid":9999}):
+                with self.assertRaisesRegex(RuntimeError, "another process"):
+                    task.start_task_owner(plugin)
+            with patch.object(task, "cleanup_reservation"):
+                task.cleanup(plugin)
+            owner.close.assert_called_once()
+
+    def test_reused_sandbox_closes_previous_owner_before_preparing_another_task(self):
+        plugin = Mock(native_owner=Mock())
+        plugin.GetPluginInfoEntry.return_value = "not-a-run-uuid"
+        with patch.object(task, "cleanup") as cleanup:
+            with self.assertRaises(ValueError):
+                task.prepare(plugin)
+        cleanup.assert_called_once_with(plugin)

web/deadline_queue.js

@@ -0,0 +1,162 @@
+import {app} from "../../scripts/app.js";
+import {api} from "../../scripts/api.js";
+import {FarmProgress, installActivePagination, installRouting} from "./deadline_routing.js";
+import {captureDeadlineDestination, inspectNativeQueue, installNativeControls} from "./native_queue_compat.js";
+import {installDeadlineSidebar} from "./deadline_sidebar.js";
+
+app.registerExtension({
+    name: "Deadline.NativeQueue",
+    async setup() {
+        const response = await api.fetchApi("/deadline/farm/capabilities");
+        if (!response.ok) return;
+        const capabilities = await response.json();
+        let configured = {};
+        const sidebar = capabilities.mode !== "prototype" ? installDeadlineSidebar(app, api, capabilities, value => { configured = value; }) : null;
+        if (capabilities.mode === "unsupported") {
+            app.extensionManager.toast.addAlert(capabilities.message);
+            return;
+        }
+        if (sidebar) {
+            try { await sidebar.model.load(); }
+            catch (error) { sidebar.model.error = error.message; }
+        }
+        if (!capabilities.ready && capabilities.mode !== "native") return;
+        let stats;
+        try {
+            const response = await api.fetchApi("/system_stats");
+            if (!response.ok) throw new Error(`System information unavailable (${response.status})`);
+            stats = await response.json();
+        } catch (error) {
+            sidebar?.setRun(null, [error.message]);
+            return;
+        }
+        const version = stats.system.comfy_package_versions.find(item => item.name === "comfyui-frontend-package")?.installed;
+        // Extension setup runs before the action bar and lazy preview store
+        // mount. Return now so ComfyUI can finish booting, then check the seam.
+        let checks = 0;
+        function initialize() {
+        const state = inspectNativeQueue(app, version);
+        if (version === "1.53.10" && state.problems.length && ++checks < 120) {
+            setTimeout(initialize, 250);
+            return;
+        }
+        app.deadlineQueue = {capabilities, problems: state.problems, stale: false};
+        if (state.problems.length) {
+            sidebar?.setRun(null, state.problems);
+            app.extensionManager.toast.addAlert(state.problems.join(" "));
+            return;
+        }
+        if (capabilities.mode === "prototype") app.extensionManager.toast.addAlert(capabilities.message);
+        let farmSelected = false;
+        let stopped = false;
+        let timer;
+        const progress = new FarmProgress();
+        const selectDestination = value => { farmSelected = value; app.deadlineQueue.selected = value; };
+        const getReadiness = () => app.deadlineQueue.stale ? {ready:false,message:"Deadline connection is unavailable. Refresh the Deadline sidebar before submitting."} : sidebar?.getReadiness() || {ready:true};
+        const getDestination = () => farmSelected && state.settings.mode === "disabled" ? {settings:configured} : null;
+        const captureDestination = () => {
+            const readiness = getReadiness();
+            if (getDestination() && !readiness.ready) app.extensionManager.toast.addAlert(readiness.message);
+            return captureDeadlineDestination(farmSelected, state.settings.mode, configured, readiness);
+        };
+        const restorePagination = installActivePagination(api);
+        const restoreRouting = installRouting(app, api, captureDestination);
+        const retryKeys = new Map();
+        const retry = async id => {
+            if (!retryKeys.has(id)) retryKeys.set(id, crypto.randomUUID());
+            const response = await api.fetchApi(`/deadline/farm/${id}/retry`, {
+                method: "POST", headers: {"Content-Type": "application/json"},
+                body: JSON.stringify({submission_key: retryKeys.get(id)})
+            });
+            if (!response.ok) { app.extensionManager.toast.addAlert(`Deadline retry failed: ${await response.text()}`); return; }
+            retryKeys.delete(id);
+            await state.queue.update();
+        };
+        const stop = async id => {
+            const response = await api.fetchApi(`/jobs/${id}/cancel`, {method:"POST"});
+            if (!response.ok) throw new Error(await response.text());
+            const {cancelled} = await response.json();
+            const event = progress.runs.get(id);
+            if (cancelled && event && !["completed", "failed", "cancelled"].includes(event.status)) {
+                progress.apply({...event, cancel_requested:true, sequence:Math.floor(event.sequence / 4) * 4 + 1});
+            }
+            controls.render();
+            try { await state.queue.update(); }
+            catch (error) { console.warn("[Deadline] Stop requested; native Jobs refresh failed:", error); }
+        };
+        const controls = installNativeControls({app, state, progress, getDestination, getReadiness, openSetup:sidebar?.openSetup, selectDestination, retry, stop});
+        sidebar?.setRun(() => {
+            if (!getReadiness().ready) { sidebar.openSetup(); return; }
+            state.settings.mode = "disabled";
+            selectDestination(true);
+            controls.render();
+            const button = document.querySelector('[data-testid="queue-button"]');
+            if (button && !button.disabled) button.click();
+        });
+        const ownedIds = new Set();
+        async function poll() {
+            try {
+                const response = await api.fetchApi("/deadline/farm/progress");
+                if (!response.ok) throw new Error(`Farm progress unavailable (${response.status})`);
+                const {runs, health, readiness} = await response.json();
+                if (stopped) return;
+                app.deadlineQueue.health = health;
+                app.deadlineQueue.readiness = readiness;
+                sidebar?.updateHealth(health, readiness);
+                if (health && !health.connected && configured.enabled) throw new Error(health.message);
+                if (stopped) return;
+                progress.restore(runs);
+                const next = {...state.execution.nodeProgressStatesByJob};
+                for (const id of ownedIds) {
+                    if (!progress.runs.has(id)) { delete next[id]; state.previews.clearPreview(id); ownedIds.delete(id); }
+                }
+                for (const event of progress.runs.values()) {
+                    ownedIds.add(event.run_id);
+                    if (event.status === "in_progress") {
+                        next[event.run_id] = event.nodes;
+                        if (event.preview_url) state.previews.setPreviewUrl(event.run_id, event.preview_url);
+                    } else {
+                        delete next[event.run_id];
+                        state.previews.clearPreview(event.run_id);
+                    }
+                }
+                // Deliberately leave activeJobId, global nodeProgressStates,
+                // canvas output store and legacy execution events untouched.
+                state.execution.nodeProgressStatesByJob = next;
+                app.deadlineQueue.stale = false;
+                controls.render();
+            } catch (error) {
+                if (!app.deadlineQueue.stale) {
+                    console.warn("[Deadline] Keeping stale farm state:", error);
+                }
+                app.deadlineQueue.stale = true;
+                sidebar?.updateHealth({connected:false,message:error.message}, {ready:false,code:"repository",message:error.message});
+                controls.render();
+            } finally {
+                try { await state.queue.update(); }
+                catch (error) { console.warn("[Deadline] Native Jobs refresh failed:", error); }
+                if (!stopped) timer = setTimeout(poll, 2000);
+            }
+        }
+        const onChanged = () => { state.queue.update(); };
+        api.addEventListener("deadline:queue-changed", onChanged);
+        const dispose = () => {
+            stopped = true;
+            clearTimeout(timer);
+            controls.dispose();
+            sidebar?.dispose();
+            restoreRouting();
+            restorePagination();
+            api.removeEventListener("deadline:queue-changed", onChanged);
+            window.removeEventListener("pagehide", dispose);
+            const next = {...state.execution.nodeProgressStatesByJob};
+            for (const id of ownedIds) { delete next[id]; state.previews.clearPreview(id); }
+            state.execution.nodeProgressStatesByJob = next;
+        };
+        window.addEventListener("pagehide", dispose);
+        app.deadlineQueue.dispose = dispose;
+        poll();
+        }
+        setTimeout(initialize, 0);
+    }
+});

web/deadline_routing.js

@@ -0,0 +1,138 @@
+const MARKER = "__deadline_native_queue";
+const DISTRIBUTED_NODES = new Set(["DistributedCollector", "DeadlineDistributedCollector", "DistributedVideoCollector",
+    "DeadlineDistributedVideoCollector", "UltimateSDUpscaleDistributed", "DeadlineUltimateSDUpscaleDistributed"]);
+
+function farmError(message) {
+    return new Response(JSON.stringify({error: {type: "deadline_submission_error", message, details: ""}, node_errors: {}}),
+        {status: 400, headers: {"Content-Type": "application/json"}});
+}
+
+// These internal boundaries are verified only for frontend 1.53.10. Route
+// tokens travel with a particular serialized workflow, never the next request.
+export function installRouting(app, api, captureDestination) {
+    const originalQueue = app.queuePrompt;
+    const originalGraph = app.graphToPrompt;
+    const originalPop = app.queueItems.pop;
+    const originalFetch = api.fetchApi;
+    const batches = new Map();
+    const requests = new Map();
+    let processingBatch;
+
+    const queue = function (...args) {
+        const destination = captureDestination();
+        if (destination) {
+            const count = args[1] ?? 1;
+            if (!Number.isInteger(count) || count < 1 || count > 10000) throw new Error("Run (Deadline) requires a finite batch count between 1 and 10000.");
+            const options = args[2];
+            const targets = Array.isArray(options) ? options : options?.queueNodeIds;
+            if (targets?.length) throw new Error("Run (Deadline) supports whole workflows only.");
+        }
+        const requestId = this.nextQueueRequestId;
+        batches.set(requestId, destination ? structuredClone({...destination, batch_id: crypto.randomUUID()}) : null);
+        return originalQueue.apply(this, args);
+    };
+    const pop = function (...args) {
+        const item = originalPop.apply(this, args);
+        processingBatch = item ? batches.get(item.requestId) : null;
+        if (item) batches.delete(item.requestId);
+        return item;
+    };
+    const graph = async function (...args) {
+        const batch = processingBatch;
+        const data = await originalGraph.apply(this, args);
+        if (!batch || !app.processingQueue) return data;
+        if (Object.values(data.output).some(node => DISTRIBUTED_NODES.has(node.class_type))) {
+            throw new Error("Use local Run for workflows containing Distributed nodes. Run (Deadline) queues whole workflows.");
+        }
+        const token = crypto.randomUUID();
+        requests.set(token, {...batch, submission_key: crypto.randomUUID(), prepared_at: Date.now()});
+        // Keep bounded retry contexts, including responses lost in another
+        // extension's transport wrapper. They carry no prompt or credentials.
+        for (const [key, context] of requests) if (Date.now() - context.prepared_at > 300000) requests.delete(key);
+        return {...data, workflow: {...data.workflow, [MARKER]: token}};
+    };
+    const fetch = async function (route, options, ...args) {
+        if (route !== "/prompt" || options?.method !== "POST") {
+            return originalFetch.call(this, route, options, ...args);
+        }
+        const body = JSON.parse(options.body);
+        const workflow = body.extra_data?.extra_pnginfo?.workflow;
+        const token = workflow?.[MARKER];
+        if (!token) return originalFetch.call(this, route, options, ...args);
+        const context = requests.get(token);
+        if (!context) return farmError("Deadline request context expired; the workflow was not queued locally.");
+        if (body.partial_execution_targets?.length) return farmError("Run (Deadline) supports whole workflows only.");
+        if (Object.values(body.prompt).some(node => ["DeadlineSubmit", "SaveAndSubmitNode"].includes(node.class_type))) {
+            return farmError("Choose local Run for workflows containing Submit to Deadline, or remove that node for Run (Deadline).");
+        }
+        delete workflow[MARKER];
+        // Browser credentials are never sent to shared farm storage.
+        delete body.extra_data.auth_token_comfy_org;
+        delete body.extra_data.api_key_comfy_org;
+        delete body.front;
+        delete body.number;
+        const response = await originalFetch.call(this, "/deadline/farm/submit", {
+            ...options, body: JSON.stringify({...body, settings: context.settings,
+                batch_id: context.batch_id, submission_key: context.submission_key})
+        }, ...args);
+        api.dispatchCustomEvent("deadline:queue-changed", {});
+        return response;
+    };
+
+    app.queuePrompt = queue;
+    app.queueItems.pop = pop;
+    app.graphToPrompt = graph;
+    api.fetchApi = fetch;
+    return () => {
+        if (app.queuePrompt === queue) app.queuePrompt = originalQueue;
+        if (app.queueItems.pop === pop) app.queueItems.pop = originalPop;
+        if (app.graphToPrompt === graph) app.graphToPrompt = originalGraph;
+        if (api.fetchApi === fetch) api.fetchApi = originalFetch;
+        requests.clear();
+        batches.clear();
+    };
+}
+
+export function installActivePagination(api, maximum = 10000) {
+    const previous = api.fetchApi;
+    const fetch = async function (route, ...args) {
+        const response = await previous.call(this, route, ...args);
+        const url = new URL(route, "http://localhost");
+        const statuses = new Set((url.searchParams.get("status") || "").split(","));
+        if (url.pathname !== "/jobs" || statuses.size !== 2 || !statuses.has("pending") ||
+            !statuses.has("in_progress") || url.searchParams.get("offset") !== "0" || !response.ok) return response;
+        const data = await response.clone().json();
+        while (data.pagination.has_more) {
+            if (data.jobs.length >= maximum) throw new Error(`Native queue exceeds the ${maximum} supported active runs; list is incomplete.`);
+            url.searchParams.set("offset", String(data.jobs.length));
+            const pageResponse = await previous.call(this, url.pathname + url.search, ...args);
+            if (!pageResponse.ok) throw new Error("Could not load the complete native active queue.");
+            const page = await pageResponse.json();
+            if (!page.jobs.length) throw new Error("Active queue pagination made no progress.");
+            data.jobs.push(...page.jobs);
+            data.pagination = page.pagination;
+        }
+        data.pagination.offset = 0;
+        data.pagination.limit = data.jobs.length;
+        data.pagination.total = data.jobs.length;
+        return new Response(JSON.stringify(data), {status: response.status, headers: response.headers});
+    };
+    api.fetchApi = fetch;
+    return () => { if (api.fetchApi === fetch) api.fetchApi = previous; };
+}
+
+export class FarmProgress {
+    constructor() { this.runs = new Map(); }
+    apply(event) {
+        const previous = this.runs.get(event.run_id);
+        if (previous && (event.generation < previous.generation ||
+            (event.generation === previous.generation && event.sequence <= previous.sequence))) return false;
+        this.runs.set(event.run_id, event);
+        return true;
+    }
+    restore(events) {
+        const present = new Set(events.map(event => event.run_id));
+        for (const id of this.runs.keys()) if (!present.has(id)) this.runs.delete(id);
+        for (const event of events) this.apply(event);
+    }
+}

web/deadline_settings.js

@@ -0,0 +1,41 @@
+const same = (a, b) => JSON.stringify(Object.keys(a).sort().map(k => [k, a[k]])) === JSON.stringify(Object.keys(b).sort().map(k => [k, b[k]]));
+
+export class DeadlineSettings {
+    constructor(api, onSaved, onChange = () => {}) {
+        Object.assign(this, {api, onSaved, onChange, saved: null, draft: {}, conflict: false, saving: false, error: "", loadVersion: 0});
+    }
+    get dirty() { return this.saved !== null && !same(this.saved, this.draft); }
+    edit(key, value) { this.draft = {...this.draft, [key]: value}; this.error = ""; this.onChange(); }
+    accept(settings) {
+        if (this.dirty || this.saving) this.conflict ||= !same(this.saved, settings);
+        else { this.draft = {...settings}; this.conflict = false; this.error = ""; }
+        this.saved = {...settings}; this.onSaved(settings); this.onChange();
+    }
+    revert() { this.draft = {...this.saved}; this.conflict = false; this.error = ""; this.onChange(); }
+    async load(signal) {
+        const version = ++this.loadVersion;
+        const response = await this.api.fetchApi("/deadline/farm/settings", {signal});
+        if (!response.ok) throw new Error("Could not load Deadline settings");
+        const value = await response.json();
+        if (signal?.aborted) throw new DOMException("Request aborted", "AbortError");
+        if (version === this.loadVersion) this.accept(value.settings);
+        return value.health;
+    }
+    async save() {
+        if (!this.saved || this.conflict || this.saving || !this.dirty) return false;
+        this.loadVersion++;
+        this.saving = true; this.error = ""; this.onChange();
+        try {
+            const result = await this.api.fetchApi("/deadline/farm/settings", {method: "POST", headers: {"Content-Type": "application/json"},
+                body: JSON.stringify({settings: this.draft, expected_settings: this.saved})});
+            const value = await result.json();
+            if (value.error?.type === "deadline_settings_conflict") {
+                this.conflict = true;
+                await this.load();
+            }
+            if (!result.ok) throw new Error(value.error?.message || "Could not save Deadline settings");
+            this.saved = {...value}; this.draft = {...value}; this.conflict = false; this.onSaved(value); return true;
+        } catch (error) { this.error = error.message; return false; }
+        finally { this.loadVersion++; this.saving = false; this.onChange(); }
+    }
+}

web/deadline_sidebar.css

@@ -0,0 +1,65 @@
+.deadline-sidebar {
+    --dl-accent: var(--p-primary-color, #b1a0ed);
+    --dl-border: var(--border-color, #3b3b40);
+    --dl-muted: var(--descrip-text, #aaaab2);
+    height: 100%; min-height: 0; display: flex; flex-direction: column;
+    background: var(--comfy-menu-bg, #202023); color: var(--input-text, #e4e4e7);
+    font: 12px/1.5 system-ui, sans-serif; overflow: hidden;
+}
+.deadline-sidebar * {box-sizing: border-box;}
+.deadline-sidebar [hidden] {display: none !important;}
+.deadline-sidebar h2, .deadline-sidebar h3, .deadline-sidebar p {margin: 0;}
+.deadline-sidebar h2 {font-size: 17px; font-weight: 600;}
+.deadline-sidebar h3 {font-size: 12px; font-weight: 600; margin-bottom: 12px;}
+.deadline-sidebar small {display: block; color: var(--dl-muted); font-size: 11px;}
+.deadline-sidebar button, .deadline-sidebar input, .deadline-sidebar select {font: inherit; color: inherit;}
+.deadline-sidebar button {border: 1px solid var(--dl-border); border-radius: 5px; background: var(--comfy-input-bg, #29292e); padding: 7px 10px; cursor: pointer;}
+.deadline-sidebar button:hover:not(:disabled) {border-color: var(--dl-accent);}
+.deadline-sidebar button:disabled {opacity: .45; cursor: default;}
+.deadline-sidebar :focus-visible {outline: 2px solid var(--dl-accent); outline-offset: 2px;}
+.deadline-sidebar input:not([type=checkbox]), .deadline-sidebar select {width: 100%; min-width: 0; padding: 7px 8px; margin-top: 5px; border: 1px solid var(--dl-border); border-radius: 5px; background: var(--comfy-input-bg, #19191d);}
+.deadline-sidebar input[type=checkbox] {width: 14px; height: 14px; margin: 3px 0 0; accent-color: var(--dl-accent); flex-shrink: 0;}
+.deadline-sidebar label {display: block; color: var(--dl-muted); font-size: 11px;}
+.dl-header {display: flex; align-items: center; justify-content: space-between; flex-shrink: 0; padding: 15px 16px; border-bottom: 1px solid var(--dl-border);}
+.dl-header button {width: 30px; height: 30px; padding: 0;}
+.dl-connection {display: flex; align-items: center; gap: 6px; color: var(--dl-muted); font-size: 10px; margin-top: 2px;}
+.dl-connection-dot {width: 5px; height: 5px; border-radius: 50%; background: #d29b5c;}
+.dl-connection[data-connected=true] .dl-connection-dot {background: #76c99d;}
+.dl-content {flex: 1; min-height: 0; overflow-y: auto; overflow-x: hidden; padding: 18px 16px;}
+.dl-grid {display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px;}
+.dl-destination-summary {font-size: 11px; margin-top: 12px !important;}
+.dl-destination-note {margin-top: 3px;}
+.dl-plugin-override {margin-top: 6px; font-size: 10px !important;}
+.dl-priority {margin-top: 23px;}
+.dl-priority label {max-width: 115px;}
+.dl-content details {border-top: 1px solid var(--dl-border); margin-top: 23px; padding-top: 12px;}
+.dl-content summary {cursor: pointer; font-size: 11px; color: var(--dl-muted);}
+.dl-more > label {margin-top: 14px;}
+.dl-content .dl-restriction {border: 0; padding-top: 0; margin-top: 18px;}
+.dl-restriction-summary {float: right; font-size: 10px;}
+.dl-restriction > small {margin-top: 10px;}
+.dl-search {margin: 8px 0 10px !important;}
+.deadline-sidebar .dl-clear-workers {padding: 0; border: 0; background: transparent; font-size: 11px; color: var(--dl-accent); margin-bottom: 10px;}
+.deadline-sidebar .dl-worker {display: flex; align-items: flex-start; gap: 9px; padding: 9px 0; border-top: 1px solid var(--dl-border); cursor: pointer;}
+.dl-worker > span:first-of-type {flex: 1; min-width: 0; overflow-wrap: anywhere;}
+.dl-worker strong {color: var(--input-text, #e4e4e7); font-size: 11px; font-weight: 500;}
+.dl-worker small {font-size: 10px; margin-top: 2px;}
+.dl-worker[data-invalid=true] small {color: #d9a768;}
+.dl-worker-status {font-size: 10px; white-space: nowrap;}
+.dl-worker-status[data-state=idle] {color: #76c99d;}
+.dl-worker-status[data-state=busy] {color: var(--dl-accent);}
+.dl-no-workers {color: var(--dl-muted); margin-top: 8px !important; font-size: 11px;}
+.dl-notice {margin-bottom: 20px; border-left: 2px solid #d9a768; padding: 2px 0 2px 10px;}
+.dl-notice strong {font-size: 12px; font-weight: 600;}
+.dl-notice p {margin-top: 4px !important; color: var(--dl-muted); font-size: 11px; overflow-wrap: anywhere;}
+.dl-guide-link {display: inline-block; color: var(--dl-accent); margin-top: 8px; font-size: 11px; text-decoration: none;}
+.deadline-sidebar .dl-review-workers {padding: 0; border: 0; background: transparent; color: var(--dl-accent); margin-top: 8px; font-size: 11px;}
+.deadline-sidebar footer {flex-shrink: 0; padding: 10px 16px 14px; border-top: 1px solid var(--dl-border);}
+.dl-buttons {display: flex; gap: 8px;}
+.dl-buttons button {min-width: 0; flex: .65; font-size: 11px;}
+.deadline-sidebar .dl-run {flex: 1.8; background: var(--dl-accent); color: #211b35; border-color: transparent; font-weight: 600;}
+.dl-run i {font-size: 10px;}
+.dl-error {font-size: 11px; color: #ef9292; overflow-wrap: anywhere; margin-bottom: 8px !important;}
+.dl-save-state:empty {display: none;}
+.dl-save-state {font-size: 10px !important; margin-bottom: 6px;}
+.dl-reload, .dl-retry-save {margin-bottom: 8px;}

web/deadline_sidebar.js

@@ -0,0 +1,254 @@
+import {DeadlineSettings} from "./deadline_settings.js";
+import {evaluateWorkers} from "./deadline_workers.js";
+
+export async function fetchFarmDiscovery(api, force, signal) {
+    const response = await api.fetchApi(`/deadline/farm/discovery${force ? "?refresh=1" : ""}`, {signal});
+    if (response.status === 404) {
+        const error = new Error("Restart ComfyUI to load the updated Deadline backend.");
+        error.restartRequired = true;
+        throw error;
+    }
+    if (!response.ok) throw new Error(`Farm discovery unavailable (${response.status})`);
+    return response.json();
+}
+
+export function farmConnectionStatus(discovery) {
+    if (discovery.restart_required) return "Restart required";
+    if (discovery.connected === null) return "Connecting";
+    return discovery.connected ? "Connected" : "Disconnected";
+}
+
+export function sidebarReadiness(settings, discovery, workers, server, dirty) {
+    if (discovery.restart_required) return {title:"Restart required", message:discovery.message, action:"refresh"};
+    if (!discovery.connected) return {title:"Farm unavailable", message:discovery.message || "Connecting to Deadline…", action:"refresh"};
+    if (dirty) return {title:"Saving changes", message:"Your next run will use the updated destination."};
+    if (!settings.root || !settings.enabled) return {title:"Farm queue is not configured", message:"Ask your farm administrator to configure shared storage and deploy the native queue plugin.", action:"guide"};
+    if (!discovery.plugin_available) return {title:"Farm plugin needs updating", message:"The native queue helpers need to be deployed by your farm administrator.", action:"guide"};
+    const invalid = workers.selected.filter(worker => !worker.eligible);
+    if (invalid.length) return {title:"Worker restriction needs attention", message:invalid.map(worker => `${worker.name}: ${worker.reasons.join(", ")}`).join("; "), action:"workers"};
+    if (!server?.ready) return {title:"Farm queue unavailable", message:server?.message || "Checking shared storage…", action:["plugin","storage","session_configuration"].includes(server?.code) ? "guide" : "refresh"};
+    return {title:"Ready", message:"Deadline schedules each workflow. Batch count and seeds follow native Run.", ready:true};
+}
+
+export function installDeadlineSidebar(app, api, capabilities, onSaved) {
+    if (!document.querySelector("link[data-deadline-sidebar]")) {
+        const css = document.createElement("link");
+        css.rel = "stylesheet";
+        css.href = new URL("./deadline_sidebar.css", import.meta.url).href;
+        css.dataset.deadlineSidebar = "";
+        document.head.append(css);
+    }
+    let panel, controller, refreshTimer, saveTimer, run;
+    let refreshing = false, generation = 0, search = "";
+    let discovery = {connected:null, workers:[], pools:[], groups:[]};
+    let readiness;
+    let problems = capabilities.mode === "unsupported" ? [capabilities.message] : [];
+    const model = new DeadlineSettings(api, onSaved, update);
+    const $ = selector => panel?.querySelector(selector);
+    const selection = () => evaluateWorkers(model.draft, discovery, {search});
+
+    function nativeReadiness() {
+        if (problems.length) return {ready:false, message:problems.join(" ")};
+        if (model.saving || model.dirty) return {ready:false, message:"Wait for Deadline settings to finish saving."};
+        if (model.conflict || model.error) return {ready:false, message:model.error || "Settings changed in another browser. Reload settings."};
+        if (!model.saved?.enabled) return {ready:false, message:"The farm queue needs administrator configuration."};
+        return readiness || {ready:false, message:"Checking the farm queue."};
+    }
+
+    function setText(selector, value) {
+        const element = $(selector);
+        if (element) element.textContent = value;
+    }
+
+    function setOptions(key, names) {
+        const select = $(`[name="${key}"]`);
+        const values = [...new Set(["none", ...names, model.draft[key]].filter(Boolean))];
+        if (select.dataset.options !== JSON.stringify(values)) {
+            select.replaceChildren(...values.map(value => {
+                const option = document.createElement("option");
+                option.value = value;
+                option.textContent = value === "none" ? `Any ${key}` : value;
+                return option;
+            }));
+            select.dataset.options = JSON.stringify(values);
+        }
+        select.value = model.draft[key] || "none";
+    }
+
+    function edit(key, value) {
+        model.edit(key, value);
+        clearTimeout(saveTimer);
+        saveTimer = setTimeout(save, 400);
+    }
+
+    async function save() {
+        if (await model.save()) await refresh(true);
+        update();
+    }
+
+    function workerRow(worker) {
+        const label = document.createElement("label");
+        label.className = "dl-worker";
+        label.dataset.worker = worker.name;
+        label.dataset.invalid = String(!worker.eligible);
+        const checkbox = document.createElement("input");
+        checkbox.type = "checkbox";
+        checkbox.checked = worker.selected;
+        checkbox.disabled = model.saving || !worker.eligible && !worker.selected;
+        checkbox.setAttribute("aria-label", `Use Worker ${worker.name}`);
+        checkbox.addEventListener("change", () => {
+            const names = (model.draft.workers || "").split(",").filter(name => name && name.toLowerCase() !== worker.name.toLowerCase());
+            if (checkbox.checked) names.push(worker.name);
+            edit("workers", names.join(","));
+        });
+        const body = document.createElement("span");
+        const name = document.createElement("strong");
+        name.textContent = worker.name;
+        const detail = document.createElement("small");
+        detail.textContent = worker.reasons.join(" · ") || [worker.host !== worker.name ? worker.host : "", worker.gpu_model].filter(Boolean).join(" · ");
+        body.append(name);
+        if (detail.textContent) body.append(detail);
+        const status = document.createElement("span");
+        status.className = "dl-worker-status";
+        status.dataset.state = worker.availability;
+        status.textContent = worker.availability === "disabled" ? "Disabled" : worker.status || "Unknown";
+        label.append(checkbox, body, status);
+        return label;
+    }
+
+    function update() {
+        if (!panel) return;
+        const workers = selection();
+        const state = sidebarReadiness(model.draft, discovery, workers, readiness, model.dirty);
+        setText(".dl-connection-label", farmConnectionStatus(discovery));
+        $(".dl-connection").dataset.connected = String(Boolean(discovery.connected));
+        $(".dl-connection").title = discovery.repository || discovery.message || "";
+        $(".dl-plugin-override").hidden = !model.saved?.plugin_directory;
+        $(".dl-plugin-override").title = model.saved?.plugin_directory || "";
+        setOptions("pool", discovery.pools);
+        setOptions("group", discovery.groups);
+        for (const input of panel.querySelectorAll("[name]")) {
+            input.disabled = !model.saved || model.saving;
+            if (input !== document.activeElement) input.value = model.draft[input.name] ?? "";
+        }
+        const destination = evaluateWorkers(model.draft, discovery);
+        const targets = model.draft.workers ? destination.selected : destination.candidates.filter(worker => worker.eligible);
+        const idle = targets.filter(worker => worker.available).length;
+        const busy = targets.filter(worker => worker.eligible && worker.availability === "busy").length;
+        setText(".dl-destination-summary", `${targets.length} Worker${targets.length === 1 ? "" : "s"} · ${idle} idle · ${busy} rendering`);
+        setText(".dl-destination-note", model.draft.workers ? `Only ${targets.slice(0,2).map(worker => worker.name).join(", ")}${targets.length > 2 ? ` + ${targets.length - 2} more` : ""}` : "Deadline chooses an available Worker.");
+        $(".dl-notice").hidden = Boolean(state.ready && !problems.length);
+        setText(".dl-notice-title", problems.length ? "Native Run unavailable" : state.title);
+        setText(".dl-notice-message", problems.length ? problems.join(" ") : state.message);
+        $(".dl-guide-link").hidden = state.action !== "guide";
+        $(".dl-review-workers").hidden = state.action !== "workers";
+        setText(".dl-restriction-summary", model.draft.workers ? `${workers.selected.length} selected` : "Optional");
+        const list = $(".dl-worker-list");
+        const fingerprint = JSON.stringify([workers.selected, workers.candidates, model.saving]);
+        if (list.dataset.fingerprint !== fingerprint) {
+            const focused = document.activeElement?.closest(".dl-worker")?.dataset.worker;
+            list.replaceChildren(...[...workers.selected, ...workers.candidates].map(workerRow));
+            list.dataset.fingerprint = fingerprint;
+            if (focused) [...list.children].find(row => row.dataset.worker === focused)?.querySelector("input")?.focus({preventScroll:true});
+        }
+        $(".dl-no-workers").hidden = Boolean(workers.selected.length || workers.candidates.length);
+        $(".dl-clear-workers").hidden = !model.draft.workers;
+        $(".dl-clear-workers").disabled = model.saving;
+        setText(".dl-save-state", model.saving || model.dirty ? "Saving…" : "");
+        const error = model.conflict ? "Settings changed in another browser. Reload before editing." : model.error;
+        setText(".dl-error", error);
+        $(".dl-error").hidden = !error;
+        $(".dl-reload").hidden = !model.conflict;
+        $(".dl-retry-save").hidden = !model.error || model.conflict;
+        $("[data-action=run]").disabled = !run || !state.ready || !nativeReadiness().ready;
+        $("[data-action=run]").title = state.ready ? "Uses native batch count and seeds" : state.message;
+        $("[data-action=refresh]").disabled = refreshing;
+    }
+
+    async function refresh(force = false) {
+        if (refreshing || !panel) return;
+        clearTimeout(refreshTimer);
+        refreshing = true;
+        update();
+        const current = generation;
+        const signal = controller.signal;
+        try {
+            const results = await Promise.allSettled([
+                model.load(signal), fetchFarmDiscovery(api, force, signal),
+                api.fetchApi("/deadline/farm/readiness", {signal}).then(async response => {
+                    if (!response.ok) throw new Error("Could not check the farm queue");
+                    return response.json();
+                })
+            ]);
+            if (current !== generation) return;
+            if (results[0].status === "rejected" && results[0].reason.name !== "AbortError") model.error = results[0].reason.message;
+            discovery = results[1].status === "fulfilled" ? results[1].value : {...discovery, connected:false, message:results[1].reason.message, restart_required:Boolean(results[1].reason.restartRequired)};
+            readiness = results[2].status === "fulfilled" ? results[2].value : {ready:false, message:results[2].reason.message};
+        } finally {
+            if (current === generation) {
+                refreshing = false;
+                update();
+                if (panel) refreshTimer = setTimeout(refresh, 15000);
+            }
+        }
+    }
+
+    function destroy() {
+        generation++;
+        controller?.abort();
+        clearTimeout(refreshTimer);
+        panel = null;
+        refreshing = false;
+    }
+
+    app.extensionManager.registerSidebarTab({
+        id:"deadline", title:"Deadline", tooltip:"Deadline render farm", icon:"pi pi-send", type:"custom",
+        render(container) {
+            destroy();
+            controller = new AbortController();
+            container.style.cssText = "height:100%;min-height:0;overflow:hidden";
+            panel = document.createElement("div");
+            panel.className = "deadline-sidebar";
+            panel.innerHTML = `
+                <header class="dl-header"><div><h2>Deadline</h2><div class="dl-connection"><span class="dl-connection-dot"></span><span class="dl-connection-label">Connecting</span></div></div><button type="button" data-action="refresh" aria-label="Refresh farm" title="Refresh farm"><i class="pi pi-refresh" aria-hidden="true"></i></button></header>
+                <div class="dl-content">
+                    <section class="dl-notice" role="status"><strong class="dl-notice-title"></strong><p class="dl-notice-message"></p><a class="dl-guide-link" target="_blank" rel="noopener">Administrator guide <i class="pi pi-external-link" aria-hidden="true"></i></a><button type="button" class="dl-review-workers" hidden>Review restriction</button></section>
+                    <section><h3>Destination</h3><div class="dl-grid"><label>Pool<select name="pool"></select></label><label>Group<select name="group"></select></label></div><p class="dl-destination-summary"></p><small class="dl-destination-note"></small><small class="dl-plugin-override" hidden>Custom render plugin active</small></section>
+                    <section class="dl-priority"><label>Priority<input name="priority" type="number" min="0" max="100"></label></section>
+                    <details class="dl-more"><summary>More options</summary><label>Task timeout (minutes)<input name="task_timeout_minutes" type="number" min="1" max="1440"></label>
+                        <details class="dl-restriction"><summary>Only these Workers <span class="dl-restriction-summary"></span></summary><small>Leave empty to let Deadline choose.</small><input class="dl-search" type="search" placeholder="Search Workers" aria-label="Search Workers"><button type="button" class="dl-clear-workers">Clear restriction</button><div class="dl-worker-list"></div><p class="dl-no-workers" hidden>No Workers match this destination.</p></details>
+                    </details>
+                </div>
+                <footer><p class="dl-error" role="alert" hidden></p><button class="dl-reload" type="button" hidden>Reload settings</button><button class="dl-retry-save" type="button" hidden>Retry saving</button><small class="dl-save-state" role="status"></small><div class="dl-buttons"><button type="button" data-action="jobs">Jobs</button><button type="button" data-action="run" class="dl-run"><i class="pi pi-play" aria-hidden="true"></i> Run on Deadline</button></div></footer>`;
+            container.replaceChildren(panel);
+            $(".dl-guide-link").href = api.apiURL("/deadline/farm/setup-guide");
+            for (const input of panel.querySelectorAll("[name]")) input.addEventListener("change", () => edit(input.name, input.type === "number" ? Number(input.value) : input.value));
+            $(".dl-search").value = search;
+            $(".dl-search").addEventListener("input", event => {search = event.target.value; update();});
+            $(".dl-clear-workers").addEventListener("click", () => edit("workers", ""));
+            $(".dl-review-workers").addEventListener("click", () => {$(".dl-more").open = true; $(".dl-restriction").open = true; $(".dl-search").focus();});
+            $(".dl-reload").addEventListener("click", () => {model.revert(); refresh(true);});
+            $(".dl-retry-save").addEventListener("click", save);
+            $("[data-action=refresh]").addEventListener("click", () => {clearTimeout(refreshTimer); refresh(true);});
+            $("[data-action=run]").addEventListener("click", () => {if (nativeReadiness().ready) run?.();});
+            $("[data-action=jobs]").addEventListener("click", () => {
+                const button = document.querySelector('[data-testid="queue-overlay-toggle"]');
+                if (button?.getAttribute("aria-pressed") !== "true") button?.click();
+            });
+            update();
+            refresh();
+        }, destroy
+    });
+    const dispose = () => {
+        destroy();
+        clearTimeout(saveTimer);
+        app.extensionManager.unregisterSidebarTab("deadline");
+        window.removeEventListener("pagehide", dispose);
+    };
+    window.addEventListener("pagehide", dispose);
+    return {model, dispose, getReadiness:nativeReadiness,
+        openSetup() {if (!panel?.isConnected) app.extensionManager.command.execute("Workspace.ToggleSidebarTab.deadline");},
+        updateHealth(_health, checked) {if (checked) readiness = checked; update();},
+        setRun(callback, errors = []) {run = callback; problems = errors; update();}
+    };
+}

web/deadline_workers.js

@@ -0,0 +1,69 @@
+const key = value => String(value || "").trim().toLowerCase();
+const matches = (names, selected) => !key(selected) || key(selected) === "none" || names.some(name => key(name) === key(selected));
+
+export function evaluateWorkers(settings, discovery, {search = ""} = {}) {
+    const known = new Map();
+    for (const worker of discovery.workers || []) {
+        if (!known.has(key(worker.name))) known.set(key(worker.name), worker);
+    }
+    const selectedNames = new Map();
+    for (const name of String(settings.workers || "").split(",")) {
+        if (key(name) && !selectedNames.has(key(name))) selectedNames.set(key(name), name.trim());
+    }
+
+    function evaluate(worker, missing = false) {
+        const pools = worker.pools || [], groups = worker.groups || [];
+        const poolMatches = matches(pools, settings.pool), groupMatches = matches(groups, settings.group);
+        const enabled = worker.enabled === true;
+        const reasonCodes = [], reasons = [];
+        const reason = (code, message) => {reasonCodes.push(code); reasons.push(message);};
+        if (missing) reason("missing", "Worker is no longer in the repository");
+        else {
+            if (!enabled) reason("disabled", "Worker is disabled in Deadline");
+            const mode = key(worker.scheduling_mode === undefined ? "AllJobs" : worker.scheduling_mode);
+            if (mode === "userjobs") {
+                const users = worker.user_job_names || [];
+                if (!users.some(user => key(user) === key(discovery.deadline_user || settings.scope)))
+                    reason("user", `Does not accept Deadline user ${discovery.deadline_user || settings.scope || "(empty)"}`);
+            } else if (mode === "machinejobs") {
+                if (!key(worker.machine_user)) reason("user", "Deadline machine user could not be verified");
+                else if (key(worker.machine_user) !== key(discovery.deadline_user || settings.scope)) reason("user", `Accepts only Deadline machine user ${worker.machine_user}`);
+            } else if (mode !== "alljobs") {
+                reason("scheduling", `Deadline scheduling mode ${worker.scheduling_mode} could not be verified`);
+            }
+            if (!poolMatches) reason("pool", `Not in pool ${settings.pool}`);
+            if (!groupMatches) reason("group", `Not in group ${settings.group}`);
+            if (worker.session_error) reason("session_error", "ComfyUI session configuration is invalid");
+        }
+        const status = key(worker.status);
+        const availability = missing ? "missing" : !enabled ? "disabled" : status === "idle" ? "idle" :
+            status === "rendering" ? "busy" : status === "offline" ? "offline" : "unknown";
+        const eligible = reasons.length === 0;
+        return {...worker, pools, groups, selected: selectedNames.has(key(worker.name)), missing,
+            matchesFilters: poolMatches && groupMatches, eligible,
+            available: eligible && availability === "idle", availability, reasons, reasonCodes};
+    }
+
+    const workers = [...known.values()].map(worker => evaluate(worker));
+    const byName = new Map(workers.map(worker => [key(worker.name), worker]));
+    const selected = [...selectedNames].map(([name, original]) => byName.get(name) || evaluate({name: original, status: "Missing"}, true));
+    const term = key(search);
+    const candidates = workers.filter(worker => !worker.selected &&
+        worker.matchesFilters &&
+        (!term || `${worker.name} ${worker.host || ""} ${worker.gpu_model || ""}`.toLowerCase().includes(term)));
+    const order = (a, b) => Number(b.available) - Number(a.available) || Number(b.eligible) - Number(a.eligible) || a.name.localeCompare(b.name);
+    selected.sort(order); candidates.sort(order);
+
+    return {selected, candidates, counts: {
+        farm: workers.length,
+        matching: workers.filter(worker => worker.matchesFilters).length,
+        eligible: workers.filter(worker => worker.eligible).length,
+        available: workers.filter(worker => worker.available).length,
+        selected: selected.length,
+        selectedEligible: selected.filter(worker => worker.eligible).length,
+        selectedAvailable: selected.filter(worker => worker.available).length,
+        missing: selected.filter(worker => worker.missing).length,
+        visible: selected.length + candidates.length,
+        hidden: workers.length - selected.filter(worker => !worker.missing).length - candidates.length
+    }};
+}

web/native_queue_compat.js

@@ -0,0 +1,248 @@
+// Frontend 1.53.10 only. No Vue component closures or installed bundles are
+// modified. The private Pinia registry and DOM assumptions are kept here.
+export function inspectNativeQueue(app, version) {
+    const stores = app.extensionManager?._p?._s;
+    const execution = stores?.get("execution");
+    const previews = stores?.get("jobPreview");
+    const queue = stores?.get("queue");
+    const settings = app.extensionManager?.queueSettings;
+    const problems = [];
+    if (version !== "1.53.10") problems.push(`Frontend ${version} is unsupported; expected 1.53.10.`);
+    if (!Array.isArray(app.queueItems) || !Number.isInteger(app.nextQueueRequestId)) problems.push("Native request association is unavailable.");
+    if (!execution?.nodeProgressStatesByJob || typeof previews?.setPreviewUrl !== "function" || typeof queue?.update !== "function") {
+        problems.push("Native per-job state stores are unavailable.");
+    }
+    if (!settings || !document.querySelector('[data-testid="queue-mode-menu-trigger"]')) problems.push("Native Run controls are unavailable.");
+    return {problems, execution, previews, queue, settings};
+}
+
+export function captureDeadlineDestination(selected, mode, settings, readiness) {
+    if (!selected || mode !== "disabled") return null;
+    if (!readiness.ready) throw new Error(readiness.message || "Deadline is not ready. Open the Deadline sidebar.");
+    return {settings: structuredClone(settings)};
+}
+
+export function installNativeControls({app, state, progress, getDestination, getReadiness, openSetup, selectDestination, retry, stop}) {
+    let frame;
+    let fallback;
+    let disposed = false;
+    let menuJobId;
+    const positions = new Map();
+    const cancelledLabels = new Map();
+    const stopping = new Set();
+    const queueButton = () => document.querySelector('[data-testid="queue-button"]');
+    const trigger = () => document.querySelector('[data-testid="queue-mode-menu-trigger"]');
+    const runMenu = () => {
+        const id = trigger()?.id;
+        return id ? document.querySelector(`[role="menu"][aria-labelledby="${CSS.escape(id)}"]`) : null;
+    };
+    const closeRunMenu = () => { trigger()?.click(); trigger()?.focus(); };
+    const schedule = () => {
+        if (disposed || frame || fallback) return;
+        frame = requestAnimationFrame(render);
+        // Webviews can suspend animation frames while their page remains
+        // active. Menu actions still need to mount in that state.
+        fallback = setTimeout(render, 100);
+    };
+
+    function render() {
+        cancelAnimationFrame(frame);
+        clearTimeout(fallback);
+        frame = undefined;
+        fallback = undefined;
+        if (disposed) return;
+        const farm = getDestination() !== null;
+        const readiness = getReadiness();
+        const button = queueButton();
+        if (button) {
+            const text = [...button.childNodes].find(node => node.nodeType === Node.TEXT_NODE && node.textContent.trim());
+            if (farm && text && text.textContent.trim() !== "Run (Deadline)") text.textContent = " Run (Deadline)";
+            if (farm && button.getAttribute("aria-label") !== "Run (Deadline)") button.setAttribute("aria-label", "Run (Deadline)");
+            if (!farm && text?.textContent.includes("Deadline")) text.textContent = " Run";
+            if (!farm && button.getAttribute("aria-label") === "Run (Deadline)") button.removeAttribute("aria-label");
+        }
+        const menu = runMenu();
+        if (menu && !menu.querySelector("[data-deadline-mode]")) {
+            const item = document.createElement("button");
+            item.type = "button";
+            item.role = "menuitem";
+            item.tabIndex = -1;
+            item.dataset.deadlineMode = "farm";
+            item.className = menu.querySelector("button")?.className || "";
+            item.textContent = "Run (Deadline)";
+            item.title = "Whole workflows; finite native batch count. Deadline priority governs ordering, including Shift+Run.";
+            item.addEventListener("click", event => {
+                event.preventDefault();
+                event.stopPropagation();
+                if (!getReadiness().ready) return;
+                state.settings.mode = "disabled";
+                selectDestination(true);
+                closeRunMenu();
+                schedule();
+            });
+            menu.append(item);
+        }
+        if (menu) {
+            const item = menu.querySelector("[data-deadline-mode]");
+            item.disabled = !readiness.ready;
+            item.title = readiness.ready ? "Whole workflows; finite native batch count. Deadline priority governs ordering, including Shift+Run." : readiness.message;
+            item.setAttribute("aria-disabled", String(!readiness.ready));
+            let setup = menu.querySelector("[data-deadline-setup]");
+            if (!readiness.ready && openSetup && !setup) {
+                setup = document.createElement("button");
+                setup.type = "button"; setup.role = "menuitem"; setup.tabIndex = -1;
+                setup.dataset.deadlineSetup = ""; setup.className = item.className;
+                setup.textContent = "Open Deadline";
+                setup.addEventListener("click", event => {
+                    event.preventDefault(); event.stopPropagation(); closeRunMenu(); openSetup();
+                });
+                menu.append(setup);
+            }
+            if (readiness.ready) setup?.remove();
+        }
+
+        for (const row of document.querySelectorAll("[data-job-id]")) {
+            const event = progress.runs.get(row.dataset.jobId);
+            const active = event && ["pending", "in_progress"].includes(event.status);
+            let stopButton = row.querySelector("[data-deadline-stop]");
+            if (active && stop && !stopButton) {
+                stopButton = document.createElement("button");
+                stopButton.type = "button";
+                stopButton.dataset.deadlineStop = row.dataset.jobId;
+                stopButton.style.cssText = "position:relative;z-index:1;flex-shrink:0;font-size:11px;line-height:22px;padding:0 7px;background:var(--bg-color,#222);border:1px solid var(--border-color,#555);border-radius:4px;cursor:pointer";
+                stopButton.addEventListener("click", async click => {
+                    click.preventDefault(); click.stopPropagation();
+                    const id = stopButton.dataset.deadlineStop;
+                    if (stopping.has(id)) return;
+                    stopping.add(id); schedule();
+                    try { await stop(id); }
+                    catch (error) { app.extensionManager.toast.addAlert(`Could not stop Deadline run: ${error.message}`); }
+                    finally { stopping.delete(id); schedule(); }
+                });
+                (row.firstElementChild || row).append(stopButton);
+            }
+            if (!active) stopButton?.remove();
+            else if (stopButton) {
+                stopButton.dataset.deadlineStop = row.dataset.jobId;
+                const requested = event.cancel_requested || stopping.has(row.dataset.jobId);
+                stopButton.disabled = !!requested;
+                const label = requested ? "Stopping…" : "Stop";
+                if (stopButton.textContent !== label) stopButton.textContent = label;
+                stopButton.setAttribute("aria-label", requested ? "Stopping Deadline run" : "Stop Deadline run");
+                stopButton.title = requested ? "Waiting for the Deadline task to stop" : "Stop this run on Deadline";
+            }
+            // This frontend groups cancelled runs in its Failed tab. Keep its
+            // API status intact and distinguish farm Stop in the native row.
+            for (const label of row.querySelectorAll('span.truncate[title]')) {
+                if (event?.status === "cancelled" && label.getAttribute("title") === "Failed") {
+                    cancelledLabels.set(label, {text: label.textContent, title: label.title, run: row.dataset.jobId});
+                    label.textContent = "Cancelled";
+                    label.title = "Cancelled";
+                } else if (cancelledLabels.has(label) && (event?.status !== "cancelled" || cancelledLabels.get(label).run !== row.dataset.jobId)) {
+                    const prior = cancelledLabels.get(label);
+                    if (label.title === "Cancelled") { label.textContent = prior.text; label.title = prior.title; }
+                    cancelledLabels.delete(label);
+                }
+            }
+            let badge = row.querySelector("[data-deadline-progress]");
+            if (!event || event.status !== "in_progress") { badge?.remove(); continue; }
+            if (!badge) {
+                badge = document.createElement("span");
+                badge.dataset.deadlineProgress = row.dataset.jobId;
+                badge.style.cssText = "position:absolute;right:44px;bottom:0;font-size:10px;line-height:12px;background:var(--bg-color,#222);padding:0 3px;pointer-events:none";
+                if (!positions.has(row)) positions.set(row, row.style.position);
+                row.style.position = "relative";
+                row.append(badge);
+            }
+            const nodes = Object.values(event.nodes || {});
+            const max = nodes.reduce((sum, node) => sum + node.max, 0);
+            const value = nodes.reduce((sum, node) => sum + node.value, 0);
+            const label = event.cancel_requested ? "Deadline stopping" : max > 0 ? `Deadline ${Math.round(100 * value / max)}%` : "Deadline initializing";
+            if (badge.textContent !== label) badge.textContent = label;
+        }
+        for (const row of positions.keys()) if (!row.isConnected) positions.delete(row);
+        for (const label of cancelledLabels.keys()) if (!label.isConnected) cancelledLabels.delete(label);
+
+        // Copy-ID is a stable, inspected native context-menu action on this
+        // supported English frontend. Unsupported menu layouts get no injection.
+        for (const anchor of document.querySelectorAll('button[aria-label="Copy job ID"]')) {
+            const menu = anchor.parentElement;
+            const event = progress.runs.get(menuJobId);
+            const terminal = event && ["completed", "failed", "cancelled"].includes(event.status);
+            let action = menu.querySelector("[data-deadline-retry]");
+            if (!terminal) { action?.remove(); continue; }
+            if (action?.dataset.deadlineRetry === menuJobId) continue;
+            action?.remove();
+            action = document.createElement("button");
+            action.type = "button";
+            action.dataset.deadlineRetry = menuJobId;
+            action.setAttribute("aria-label", "Retry on Deadline");
+            action.className = anchor.className;
+            action.textContent = "Retry on Deadline";
+            const id = menuJobId;
+            action.addEventListener("click", async event => {
+                event.stopPropagation();
+                action.disabled = true;
+                try { await retry(id); } finally { action.disabled = false; }
+            });
+            menu.append(action);
+        }
+    }
+
+    function captureClick(event) {
+        const row = event.target.closest?.("[data-job-id]");
+        if (row) menuJobId = row.dataset.jobId;
+        const menu = runMenu();
+        if (menu?.contains(event.target) && !event.target.closest("[data-deadline-mode],[data-deadline-setup]")) {
+            selectDestination(false);
+            const button = queueButton();
+            const text = [...(button?.childNodes || [])].find(node => node.nodeType === Node.TEXT_NODE && node.textContent.trim());
+            if (text?.textContent.includes("Deadline")) text.textContent = " Run";
+            button?.removeAttribute("aria-label");
+            schedule();
+        }
+    }
+    function keydown(event) {
+        const menu = runMenu();
+        if (!menu?.contains(event.target)) return;
+        const items = [...menu.querySelectorAll('[role="menuitem"]')].filter(item => !item.disabled);
+        const index = items.indexOf(event.target);
+        const extensionItem = item => item?.matches("[data-deadline-mode],[data-deadline-setup]");
+        if (index >= 0 && ["ArrowDown", "ArrowUp"].includes(event.key)) {
+            const next = items[(index + (event.key === "ArrowDown" ? 1 : -1) + items.length) % items.length];
+            if (extensionItem(event.target) || extensionItem(next)) {
+                event.preventDefault(); event.stopImmediatePropagation(); next.focus();
+            }
+        } else if (extensionItem(event.target) && ["Enter", " "].includes(event.key)) {
+            event.preventDefault(); event.stopImmediatePropagation(); event.target.click();
+        }
+    }
+
+    const observer = new MutationObserver(schedule);
+    observer.observe(document.body, {childList: true, subtree: true, characterData: true});
+    document.addEventListener("click", captureClick, true);
+    document.addEventListener("contextmenu", captureClick, true);
+    document.addEventListener("keydown", keydown, true);
+    schedule();
+    return {
+        render: schedule,
+        dispose() {
+            disposed = true;
+            observer.disconnect();
+            cancelAnimationFrame(frame);
+            clearTimeout(fallback);
+            document.removeEventListener("click", captureClick, true);
+            document.removeEventListener("contextmenu", captureClick, true);
+            document.removeEventListener("keydown", keydown, true);
+            for (const element of document.querySelectorAll("[data-deadline-mode],[data-deadline-setup],[data-deadline-progress],[data-deadline-retry],[data-deadline-stop]")) element.remove();
+            for (const [row, position] of positions) row.style.position = position;
+            for (const [label, prior] of cancelledLabels) {
+                if (label.title === "Cancelled") { label.textContent = prior.text; label.title = prior.title; }
+            }
+            const button = queueButton();
+            const text = [...(button?.childNodes || [])].find(node => node.nodeType === Node.TEXT_NODE && node.textContent.includes("Deadline"));
+            if (text) text.textContent = " Run";
+            button?.removeAttribute("aria-label");
+        }
+    };
+}