feat: integrate native image-list caching and exact serverless value overrides

This commit is contained in:
Robert Wojciechowski
2026-10-14 04:08:48 +00:00
55 changed files with 4414 additions and 1112 deletions
@@ -0,0 +1,172 @@
"""Offline CPU proof against ComfyUI's real signatures and PromptExecutor.
Exercises one native worker executor and separate private master-slot lifetimes,
including their shared stable Start / fresh End contract.
"""
import asyncio
import copy
import importlib
from pathlib import Path
import sys
import types
ROOT = Path(__file__).resolve().parents[2]
def main(core):
sys.path.insert(0, str(Path(core).resolve()))
import comfy.cli_args
comfy.cli_args.args.cpu = True # Must precede all model-management imports.
comfy.cli_args.args.disable_assets = True
import execution
import nodes
import torch
from comfy_execution.caching import CacheKeySetInputSignature
name = '_native_cache_identity_probe'
for suffix in ('', '.image_list', '.nodes', '.utils'):
package = types.ModuleType(name + suffix)
package.__path__ = [str(ROOT / suffix.lstrip('.').replace('.', '/'))]
sys.modules[name + suffix] = package
def forbidden(*args, **kwargs):
raise AssertionError('offline native proof attempted a server-loop bridge')
sys.modules[name + '.utils.async_helpers'] = types.SimpleNamespace(run_async_in_server_loop=forbidden)
rt = importlib.import_module(name + '.image_list.runtime')
boundary = importlib.import_module(name + '.nodes.image_list')
calls = []
class Operation:
@classmethod
def INPUT_TYPES(cls):
return {'required': {'image': ('IMAGE',), 'mask': ('MASK',),
'text': ('STRING', {'forceInput': True})}}
RETURN_TYPES = ('IMAGE', 'MASK', 'STRING')
FUNCTION = 'run'
def run(self, image, mask, text):
calls.append(text)
return image + 1, mask + 1, text + ':done'
nodes.NODE_CLASS_MAPPINGS.update(DistributedImageListStart=boundary.DistributedImageListStart,
DistributedImageListEnd=boundary.DistributedImageListEnd,
Operation=Operation)
prompt = {
'start': {'class_type': rt.START_CLASS, 'inputs': {'image_list_execution_id': 'outer-A'}},
'op': {'class_type': 'Operation', 'inputs': {
'image': ['start', 0], 'mask': ['start', 1], 'text': ['start', 4]}},
'end': {'class_type': rt.END_CLASS, 'inputs': {
'images': ['op', 0], 'masks': ['op', 1], 'texts': ['op', 2],
'context': ['start', 3], 'value_3': ['start', 2]}},
}
data = rt.normalize_image_list_inputs(torch.zeros(1, 2, 3, 3, dtype=torch.float64),
torch.ones(1, 2, 3, dtype=torch.bfloat16),
{'value_1': 'original', 'value_2': 3})[0]
region = rt.validate_image_list_regions(prompt)[0]
def prepare(data):
token = rt.create_worker_task_token()
context = rt.create_master_job('start', 'end', prompt, [data], ['worker'], '')
rt.register_assignment(context, data['index'], 'worker', token)
identity = rt.store_worker_item(token, rt.encode_item(data))
result = rt.build_worker_prompt(prompt, region, token, context, data['index'], '', item_identity=identity)
result['start']['inputs']['image_list_local_execution'] = True
result['end']['inputs']['image_list_local_execution'] = True
return token, context, result
async def signature(graph, executor):
dynamic = execution.DynamicPrompt(graph)
changed = execution.IsChangedCache('signature-only', dynamic, executor.caches.outputs)
keys = CacheKeySetInputSignature(dynamic, graph.keys(), changed)
await keys.add_keys(graph.keys())
return {node: keys.get_data_key(node) for node in graph}
async def probe():
server = types.SimpleNamespace(client_id=None, last_node_id=None, send_sync=lambda *a, **kw: None)
executor = execution.PromptExecutor(server, cache_type=execution.CacheType.LRU,
cache_args={'lru': 64, 'ram': 0, 'ram_inactive': 0})
ta, ca, a = prepare(data)
prompt['start']['inputs']['image_list_execution_id'] = 'outer-B'
tb, cb, b = prepare(data)
assert ta != tb and ca != cb
ka, kb = await signature(a, executor), await signature(b, executor)
assert ka['start'] == kb['start'] and ka['op'] == kb['op']
assert ka['end'] != kb['end']
assert a['start']['inputs'] == b['start']['inputs']
# Exercise actual V3 list mapping, hidden.prompt/unique_id, lazy inputs,
# native node signatures and cache hits, not a reimplementation of them.
for number, (token, context, graph) in enumerate(((ta, ca, a), (tb, cb, b))):
valid = await execution.validate_prompt(f'native-{number}', graph, ['end'])
assert valid[0], valid
await executor.execute_async(graph, f'native-{number}', {}, ['end'])
assert executor.success, executor.status_messages
assert token not in rt._WORKER_ITEMS
result = rt.get_master_job(context).results[0]
assert torch.equal(result['image'], data['image'] + 1)
assert torch.equal(result['mask'], data['mask'] + 1)
assert result['text'] == 'original:done'
assert result['values'] == {'value_3': 0}
cached = next(entry['nodes'] for event, entry in executor.status_messages if event == 'execution_cached')
if number:
assert 'start' in cached and 'op' in cached and 'end' not in cached, cached
assert calls == ['original'], calls
for failure in ('expired', 'mismatched'):
token, context, graph = prepare(data)
expires, staged = rt._WORKER_ITEMS[token]
if failure == 'expired':
rt._WORKER_ITEMS[token] = (0, staged)
else:
staged['image'] = staged['image'] + 9
await executor.execute_async(graph, f'rejected-{failure}', {}, ['end'])
assert not executor.success, failure
assert not rt.get_master_job(context).received_indexes
assert token not in rt._WORKER_ITEMS
cached = next(entry['nodes'] for event, entry in executor.status_messages if event == 'execution_cached')
assert 'start' in cached and 'op' in cached and 'end' not in cached, cached
assert calls == ['original'], calls
for part in ('image', 'mask', 'value', 'index'):
changed = copy.deepcopy(data)
if part in ('image', 'mask'):
changed[part].reshape(-1)[0] += 1
elif part == 'value':
changed['values']['value_2'] = 4 # Even an unconnected slot is content.
else:
changed['index'] = 1
token, context, graph = prepare(changed)
keys = await signature(graph, executor)
assert keys['start'] != ka['start'] and keys['op'] != ka['op'], part
await executor.execute_async(graph, f'changed-{part}', {}, ['end'])
assert executor.success, executor.status_messages
assert token not in rt._WORKER_ITEMS
assert rt.get_master_job(context).received_indexes == {changed['index']}
assert len(calls) == 5, calls
# Combine the real Start/End identity path with cache transfer between
# private master slots, not just independent signature/retention probes.
local = importlib.import_module(name + '.image_list.local_executor')
before = len(calls)
for number in range(2):
token, context, graph = prepare(data)
slot = local.LocalExecutionSlot()
try:
await asyncio.wrap_future(slot.submit(graph, f'local-{number}', 'end'))
result = rt.get_master_job(context).results[0]
assert torch.equal(result['image'], data['image'] + 1)
assert result['values'] == {'value_3': 0}
assert token not in rt._WORKER_ITEMS
finally:
await slot.close()
rt.remove_master_job(context)
assert local._RETAINED_OUTPUTS is not None
assert len(calls) == before + 1, calls
assert not rt._WORKER_ITEMS
assert not torch.cuda.is_initialized()
print('NATIVE_CACHE_IDENTITY_OK stable Start/op keys; fresh End key; '
'1 operation for 2 same-data jobs; 4 changed-content misses; '
'expired/mismatched cached inputs rejected; current delivery and cleanup')
asyncio.run(probe())
if __name__ == '__main__':
main(sys.argv[1])
+128 -25
View File
@@ -37,14 +37,14 @@ import aiohttp
torch.set_num_threads(1)
SOURCE_CALLS, BRANCH_CALLS, CAPTURED, MODEL_CALLS = [], [], [], []
UTILITY_CALLS, ITEM_EXECUTIONS = [], []
UTILITY_CALLS, ITEM_EXECUTIONS, VALUE_CAPTURES = [], [], []
ALLOW_MASTER = False
COUNT = 7
def source_rows():
return [{'image': torch.full((1, 5 + i, 7, 3 if i % 2 == 0 else 4), i / 16, dtype=torch.float32),
'reference_image': torch.full((1, 2, 3, 3), (i + 1) / 16),
'reference_image': torch.full((2, 2, 3, 3), (i + 1) / 16),
'mask': torch.full((1, 5 + i, 7), (i + 1) / 16),
'text': f'item-{i}', 'strength': (i + 1) / 16, 'seed': 100 + i, 'index': i}
for i in range(COUNT)]
@@ -92,7 +92,9 @@ def assert_item_execution(prompt, index):
from comfy_execution.utils import get_executing_context
context = get_executing_context()
assert context is not None
assert prompt['2']['inputs'].get('worker_item_token'), 'per-item node ran in outer graph'
assert prompt['4']['inputs'].get('worker_result_token'), 'per-item node ran in outer graph'
assert (prompt['2']['inputs'].get('worker_item_identity')
or prompt['2']['inputs'].get('worker_item_token')), 'per-item source binding missing'
running, _ = server.PromptServer.instance.prompt_queue.get_current_queue_volatile()
assert len(running) == 1, running
entry = running[0]
@@ -160,17 +162,32 @@ class TestImageOperation:
return image + strength, mask * 0.5, f'{text}:seed-{seed}'
class TestValueOverride:
@classmethod
def INPUT_TYPES(cls):
return {'required': {'text': ('STRING', {'forceInput': True}),
'index': ('INT', {'forceInput': True})}}
RETURN_TYPES = ('STRING', 'INT')
FUNCTION = 'run'
def run(self, text, index):
assert WORKER or ALLOW_MASTER, 'override branch executed on delegate-only master'
return (text + ':override', 200 + index)
class TestCapture:
@classmethod
def INPUT_TYPES(cls):
return {'required': {'images': ('IMAGE',), 'masks': ('MASK',), 'texts': ('STRING', {'forceInput': True})}}
return {'required': {'images': ('IMAGE',), 'masks': ('MASK',), 'texts': ('STRING', {'forceInput': True})},
'optional': {'values': ('DISTRIBUTED_IMAGE_LIST_VALUES',),
'picked': ('STRING', {'forceInput': True})}}
INPUT_IS_LIST = True
OUTPUT_NODE = True
RETURN_TYPES = ()
FUNCTION = 'run'
def run(self, images, masks, texts):
def run(self, images, masks, texts, values=None, picked=None):
assert not WORKER, 'post-End capture leaked into a worker prompt'
CAPTURED.append((images, masks, texts))
VALUE_CAPTURES.append((values, picked))
return ()
@@ -179,11 +196,19 @@ async def setup():
ps = server.PromptServer(asyncio.get_running_loop(), assets)
assert await nodes.load_custom_node(str(ROOT))
module = sys.modules[str(ROOT).replace('.', '_x_')]
boundary = importlib.import_module(module.__name__ + '.nodes.image_list')
encode_result = boundary.encode_result
def checked_encode(result, **options):
encoded = encode_result(result, **options)
if 'values' in encoded:
assert set(encoded['values']) == {f'value_{n}' for n in (2, 3, 4, 5, 7, 8, 9, 10)}
return encoded
boundary.encode_result = checked_encode
nodes.NODE_CLASS_MAPPINGS.update(TestImageSource=TestImageSource,
TestImageOperation=TestImageOperation, TestCapture=TestCapture,
TestCachedModel=TestCachedModel, TestSharedFlag=TestSharedFlag,
TestRequiredOutputUtility=TestRequiredOutputUtility,
TestDiscardedPreview=TestDiscardedPreview)
TestDiscardedPreview=TestDiscardedPreview, TestValueOverride=TestValueOverride)
# Match the CORS-enabled direct-worker launch contract for canvas probes.
app = web.Application(middlewares=[server.create_cors_middleware('*')])
# Register the actual feature routes, not test replacements.
@@ -221,7 +246,10 @@ async def worker_main():
accepted = json.loads(response.body)
_, pending = ps.prompt_queue.get_current_queue_volatile()
entry = next(item for item in pending if item[1] == accepted['prompt_id'])
submissions.append({'prompt_id': entry[1], 'requested': data.get('partial_execution_targets'),
delivery = next(node['inputs'] for node in entry[2].values()
if node['class_type'] == 'DistributedImageListEnd')
submissions.append({'prompt_id': entry[1], 'item_index': delivery['worker_item_index'],
'requested': data.get('partial_execution_targets'),
'queued': entry[4], 'node_ids': sorted(entry[2]),
'end_ids': [key for key, node in entry[2].items()
if node['class_type'] == 'DistributedImageListEnd']})
@@ -271,21 +299,45 @@ def graph(callback, worker_ids):
'images': ['1', 0], 'mask': ['1', 2],
'values.value_1': ['1', 1], 'values.value_2': ['1', 3],
'values.value_3': ['1', 4], 'values.value_4': ['1', 5], 'values.value_5': ['7', 0],
'values.value_6': ['1', 2],
'image_list_enabled_worker_ids': json.dumps(worker_ids), 'image_list_master_callback': callback}},
'3': {'class_type': 'TestImageOperation', 'inputs': {
key: ['2', index] for key, index in [('image', 0), ('mask', 1), ('index', 2),
('reference_image', 4), ('text', 5), ('strength', 6), ('seed', 7), ('flag', 8)]}
| {'image': ['8', 0], 'model': ['6', 0]}},
'4': {'class_type': 'DistributedImageListEnd', 'inputs': {
'images': ['3', 0], 'masks': ['3', 1], 'texts': ['3', 2], 'context': ['2', 3]}},
'5': {'class_type': 'TestCapture', 'inputs': {'images': ['4', 0], 'masks': ['4', 1], 'texts': ['4', 2]}},
'images': ['3', 0], 'masks': ['3', 1], 'texts': ['3', 2], 'context': ['2', 3],
'value_2': ['10', 0], 'value_3': ['2', 6], 'value_4': ['2', 7], 'value_5': ['2', 8],
'value_7': ['3', 1], 'value_8': ['2', 4], 'value_9': ['2', 9], 'value_10': ['10', 1]}},
'5': {'class_type': 'TestCapture', 'inputs': {'images': ['4', 0], 'masks': ['4', 1], 'texts': ['4', 2],
'values': ['4', 3], 'picked': ['get', 0]}},
'6': {'class_type': 'TestCachedModel', 'inputs': {}},
'7': {'class_type': 'TestSharedFlag', 'inputs': {}},
'8': {'class_type': 'TestRequiredOutputUtility', 'inputs': {'image': ['2', 0], 'index': ['2', 2]}},
'9': {'class_type': 'TestDiscardedPreview', 'inputs': {'image': ['8', 0]}},
'10': {'class_type': 'TestValueOverride', 'inputs': {'text': ['2', 5], 'index': ['2', 2]}},
'get': {'class_type': 'DistributedImageListGetValue', 'inputs': {'values': ['4', 3], 'value_name': 'value_2'}},
}
def check_carried_values():
values, picked = VALUE_CAPTURES[-1]
assert len(values) == len(picked) == COUNT
for index, (actual, source) in enumerate(zip(values, source_rows())):
assert set(actual) == {f'value_{n}' for n in range(1, 11)}, actual.keys()
assert torch.equal(actual['value_1'], source['reference_image'])
assert actual['value_2'] == picked[index] == f'item-{index}:override'
assert actual['value_3'] == source['strength']
assert actual['value_4'] == source['seed']
assert actual['value_5'] is True
assert torch.equal(actual['value_6'], source['mask'])
assert torch.equal(actual['value_7'], source['mask'] * 0.5)
assert torch.equal(actual['value_8'], source['reference_image'])
assert actual['value_8'].shape[0] == 2, 'value tensor batch was split'
assert torch.equal(actual['value_9'], source['mask'])
assert actual['value_10'] == 200 + index
async def coordinator():
global ALLOW_MASTER
ps, assets, module, app = await setup()
@@ -403,6 +455,7 @@ async def coordinator():
assert executor.success, executor.status_messages
assert len(SOURCE_CALLS) == 1 and not BRANCH_CALLS and not UTILITY_CALLS
assert len(CAPTURED) == 1
check_carried_values()
images, masks, texts = CAPTURED[0]
assert len(images) == len(masks) == len(texts) == COUNT
for i, row in enumerate(source_rows()):
@@ -430,15 +483,20 @@ async def coordinator():
# left unconnected. Region N+1 must not race the prior worker End.
chained = {'1': {'class_type': 'EmptyImage', 'inputs': {
'height': 8, 'width': 8, 'batch_size': 2, 'color': 0}}}
chained['constant'] = {'class_type': 'TestSharedFlag', 'inputs': {}}
previous = '1'
value_source = 'constant'
for start, end in [('2', '3'), ('4', '5'), ('6', '7')]:
chained[start] = {'class_type': 'DistributedImageListStart', 'inputs': {
'images': [previous, 0]}}
'images': [previous, 0], 'values.value_1': [value_source, 0]}}
chained[end] = {'class_type': 'DistributedImageListEnd', 'inputs': {
'images': [start, 0], 'context': [start, 3]}}
value_source = f'get-{end}'
chained[value_source] = {'class_type': 'DistributedImageListGetValue', 'inputs': {
'values': [end, 3], 'value_name': 'value_1'}}
previous = end
chained['8'] = {'class_type': 'TestCapture', 'inputs': {
'images': ['7', 0], 'masks': ['7', 1], 'texts': ['7', 2]}}
'images': ['7', 0], 'masks': ['7', 1], 'texts': ['7', 2], 'values': ['7', 3]}}
saved_chained = copy.deepcopy(chained)
with patch.object(config_mod, 'load_config', return_value=config), \
patch.object(orchestration, 'load_config', return_value=config), \
@@ -451,6 +509,7 @@ async def coordinator():
assert len(CAPTURED) == 2 and len(CAPTURED[1][0]) == 2
assert all(torch.equal(image, torch.zeros((1, 8, 8, 3))) for image in CAPTURED[1][0])
assert CAPTURED[1][1:] == ([None, None], [None, None])
assert VALUE_CAPTURES[1][0] == [{'value_1': True}, {'value_1': True}]
assert not runtime._MASTER_JOBS
print('IMAGE_LIST_SEQUENTIAL_OK three regions; one worker; optional outputs omitted')
# Same public queue path, now with master participation enabled. The
@@ -466,6 +525,8 @@ async def coordinator():
if fmt != 'exact':
case_prompt['4']['inputs'].update(transfer_format=fmt, transfer_quality=73)
saved_case = copy.deepcopy(case_prompt)
# Keep cold-run exact-once assertions independent of other modes.
importlib.import_module(module.__name__ + '.image_list.local_executor')._RETAINED_OUTPUTS = None
executor = execution.PromptExecutor(ps, cache_args={'ram': 0, 'ram_inactive': 0}, asset_manager=assets)
before_sources, before_local, before_models = len(SOURCE_CALLS), len(BRANCH_CALLS), len(MODEL_CALLS)
before_utilities, before_events = len(UTILITY_CALLS), len(ITEM_EXECUTIONS)
@@ -489,7 +550,7 @@ async def coordinator():
assert len({event['prompt_id'] for event in ITEM_EXECUTIONS[before_events:]}) == len(local_calls)
assert local_calls, f'{mode}: master did not process any items'
assert len(MODEL_CALLS) == before_models + 1, 'local model dependency did not stay cached'
remote_calls = []
remote_assignments = []
async with aiohttp.ClientSession() as session:
for worker, before_stats in zip(workers, before_worker_stats):
async with session.get(f"http://127.0.0.1:{worker['port']}/test-stats") as response:
@@ -499,33 +560,69 @@ async def coordinator():
submissions = current['submissions'][len(before_stats['submissions']):]
events = current['item_executions'][len(before_stats['item_executions']):]
assert Counter(utilities) == Counter(calls), (mode, current)
assert len(submissions) == len(calls) == len({event['prompt_id'] for event in events})
assigned = [submission['item_index'] for submission in submissions]
# A repeated item can hit native processing cache even
# when End changes return format. Count fresh deliveries,
# not recomputation, for exact-once scheduling evidence.
assert Counter(calls) <= Counter(assigned), (mode, worker['id'], calls, assigned)
assert len(calls) == len({event['prompt_id'] for event in events}), (mode, events)
assert {event['prompt_id'] for event in events} <= {
submission['prompt_id'] for submission in submissions}
for submission in submissions:
assert submission['requested'] == submission['queued'] == ['4'], submission
assert '8' in submission['node_ids'] and '9' not in submission['node_ids']
assert '5' not in submission['node_ids']
remote_calls.extend(calls)
remote_assignments.extend(assigned)
assert not current['running'] and not current['pending'], current
if mode.startswith('mixed'):
assert remote_calls, 'mixed execution failed to use direct workers'
assert remote_assignments, 'mixed execution failed to use direct workers'
else:
assert not remote_calls and len(local_calls) == COUNT
assert Counter(local_calls + remote_calls) == Counter(range(COUNT)), (local_calls, remote_calls)
assert not remote_assignments and len(local_calls) == COUNT
assert Counter(local_calls + remote_assignments) == Counter(range(COUNT)), (local_calls, remote_assignments)
check_carried_values()
images, masks, texts = CAPTURED[-1]
assert len(images) == len(masks) == len(texts) == COUNT
for i, row in enumerate(source_rows()):
expected = row['image'] + row['strength']
if fmt == 'png' and i in remote_calls:
if fmt == 'png' and i in remote_assignments:
expected = expected.clone()
expected[..., :3] = (expected[..., :3].clamp(0, 1) * 255).to(torch.uint8).float() / 255
assert torch.equal(images[i], expected), (mode, i)
assert torch.equal(masks[i], row['mask'] * 0.5), (mode, i)
assert texts[i] == f'item-{i}:seed-{100 + i}', (mode, texts)
assert not runtime._MASTER_JOBS and not runtime._WORKER_ITEMS
participation_counts[mode] = {'master': len(local_calls), 'workers': len(remote_calls)}
participation_counts[mode] = {'master': len(local_calls), 'workers': len(remote_assignments)}
# Repeat the completed master-only queue with a fresh outer executor.
# Every item must be delivered again without recomputing its branch.
before_sources, before_local = len(SOURCE_CALLS), len(BRANCH_CALLS)
before_models, before_utilities = len(MODEL_CALLS), len(UTILITY_CALLS)
before_captures = len(CAPTURED)
previous_outputs = CAPTURED[-1]
executor = execution.PromptExecutor(ps, cache_args={'ram': 0, 'ram_inactive': 0}, asset_manager=assets)
with patch.object(config_mod, 'load_config', return_value=config), \
patch.object(orchestration, 'load_config', return_value=config), \
patch.object(orchestration, 'queue_prompt_payload', enqueue), \
patch.object(config_mod, 'get_worker_timeout_seconds', return_value=15):
await asyncio.wait_for(orchestration.orchestrate_distributed_execution(
prompt, None, 'test', []), 45)
assert executor.success, executor.status_messages
assert len(SOURCE_CALLS) == before_sources + 1
assert len(BRANCH_CALLS) == before_local
assert len(MODEL_CALLS) == before_models
assert len(UTILITY_CALLS) == before_utilities
assert len(CAPTURED) == before_captures + 1
for before, after in zip(previous_outputs[:2], CAPTURED[-1][:2]):
assert len(before) == len(after) == COUNT
assert all(torch.equal(a, b) for a, b in zip(before, after))
assert previous_outputs[2] == CAPTURED[-1][2]
check_carried_values()
assert not runtime._MASTER_JOBS and not runtime._WORKER_ITEMS
assert prompt == saved_prompt
print('IMAGE_LIST_CACHE_REQUEUE_OK unchanged items reused; fresh ordered delivery and cleanup')
print('IMAGE_LIST_MASTER_OK ' + json.dumps(participation_counts))
assert len(outer_executions) == 2 + len(participation_counts) and all(row['captures'] == 1 for row in outer_executions)
assert len(outer_executions) == 3 + len(participation_counts) and all(row['captures'] == 1 for row in outer_executions)
print('IMAGE_LIST_TRANSFER_HTTP_OK mixed-png and master-only-png; masks/text/alpha/local images exact')
print('IMAGE_LIST_VALUES_OK passthrough all six types; sparse override/add; Get; sequential Get-to-Start; direct/mixed/master-only')
print('IMAGE_LIST_TARGETS_OK ' + json.dumps({
'modes': ['delegate-only', 'mixed', 'master-only'],
'outer_targets': outer_executions[0]['targets'], 'worker_targets': ['4'],
@@ -539,6 +636,12 @@ async def coordinator():
del direct_prompt['8'], direct_prompt['9']
direct_prompt['3']['inputs']['image'] = ['2', 0]
# Real Comfy user-interrupt flag while a real worker executor is active.
rows = source_rows()
items = runtime.normalize_image_list_inputs(
[row['image'] for row in rows], [row['mask'] for row in rows],
{f'value_{n}': [row[key] for row in rows]
for n, key in ((1, 'reference_image'), (2, 'text'), (3, 'strength'), (4, 'seed'), (6, 'mask'))}
| {'value_5': True})
slow = workers[1]
async with aiohttp.ClientSession() as session:
async def slow_stats():
@@ -552,11 +655,11 @@ async def coordinator():
await asyncio.sleep(0.02)
raise AssertionError('worker state transition timed out')
before = await until(lambda state: not state['running'] and not state['pending'])
context = runtime.create_master_job('2', '4', direct_prompt, source_rows(), ['slow'],
context = runtime.create_master_job('2', '4', direct_prompt, items, ['slow'],
f'http://127.0.0.1:{port}')
with patch.object(config_mod, 'load_config', return_value=config), \
patch.object(config_mod, 'get_worker_timeout_seconds', return_value=15):
task = asyncio.create_task(runtime.run_image_list_job(context))
task = asyncio.create_task(runtime.run_image_list_job(context, include_values=True))
active = await until(lambda state: bool(state['running']))
owned_id = active['running'][0]
comfy.model_management.interrupt_current_processing(True)
@@ -574,14 +677,14 @@ async def coordinator():
assert not runtime._MASTER_JOBS
print('IMAGE_LIST_CANCEL_OK owned prompt stopped; no further dispatch; context removed')
# A worker exception before End must fail, not hang or return a partial list.
bad_rows = source_rows()
bad_rows[0]['seed'] = -1
bad_rows = copy.deepcopy(items)
bad_rows[0]['values']['value_4'] = -1
context = runtime.create_master_job('2', '4', direct_prompt, bad_rows, ['fast'],
f'http://127.0.0.1:{port}')
with patch.object(config_mod, 'load_config', return_value=config), \
patch.object(config_mod, 'get_worker_timeout_seconds', return_value=15):
try:
await asyncio.wait_for(runtime.run_image_list_job(context), 5)
await asyncio.wait_for(runtime.run_image_list_job(context, include_values=True), 5)
raise AssertionError('worker exception was ignored')
except RuntimeError as exc:
assert 'failed image-list item' in str(exc), str(exc)
@@ -0,0 +1,169 @@
"""Real CPU PromptExecutor + standalone bridge proof, no provider or service."""
import asyncio
import importlib
import json
from pathlib import Path
import shutil
import struct
import sys
import types
from media_end_to_end_check import bootstrap
ROOT = Path(__file__).resolve().parents[2]
async def check(base):
import torch
import execution
import folder_paths
import nodes
import server
from app.assets.manager import default_asset_manager
from comfy_api.v0_0_2 import io
torch.set_num_threads(1)
assert not torch.cuda.is_initialized()
assert execution.comfy.model_management.get_torch_device().type == 'cpu'
# Deploy ONLY the bridge folder to a separate import root. Import it before
# master code: it cannot rely on the Distributed image_list package.
shutil.copytree(ROOT / 'serverless_bridge', base / 'endpoint_bridge')
sys.path.insert(0, str(base))
previously_loaded = set(sys.modules)
bridge = importlib.import_module('endpoint_bridge')
handler = importlib.import_module('endpoint_bridge.handler')
extension = await bridge.comfy_entrypoint()
classes = await extension.get_node_list()
assert not any('.image_list' in name for name in set(sys.modules) - previously_loaded)
calls = []
class CalculateOverrides(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id='CalculateOverrides', inputs=[io.Image.Input('image'), io.Mask.Input('mask'),
*[io.AnyType.Input(name) for name in ('integer', 'boolean', 'floating', 'text', 'reference', 'reference_mask')]],
outputs=[io.Image.Output(), io.Mask.Output(), io.String.Output(), io.Int.Output(),
io.Float.Output(), io.Boolean.Output()])
@classmethod
def execute(cls, image, mask, integer, boolean, floating, text, reference, reference_mask):
assert type(integer) is int and type(boolean) is bool and type(floating) is float
calls.append(integer)
return io.NodeOutput(reference + image, reference_mask + mask,
text + ':calculated', integer + 7, floating / 2, not boolean)
class MustNotExecute(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(node_id='MustNotExecute', inputs=[io.Image.Input('images')], outputs=[], is_output_node=True)
@classmethod
def execute(cls, images):
raise AssertionError('unrelated output was not pruned')
for cls in [*classes, CalculateOverrides, MustNotExecute]:
cls.GET_NODE_INFO_V1()
nodes.NODE_CLASS_MAPPINGS[cls.GET_SCHEMA().node_id] = cls
package = '_serverless_value_master'
for suffix in ('', '.api', '.api.orchestration', '.image_list'):
module = types.ModuleType(package + suffix)
module.__path__ = [str(ROOT.joinpath(*suffix.lstrip('.').split('.')))] if suffix else [str(ROOT)]
sys.modules[module.__name__] = module
adapter = importlib.import_module(package + '.image_list.serverless')
decode = importlib.import_module(package + '.api.orchestration.serverless_typed_results').collect_typed_results
rt = adapter.runtime
assets = default_asset_manager()
ps = server.PromptServer(asyncio.get_running_loop(), assets)
def node(kind, **inputs):
return {'class_type': kind, 'inputs': inputs}
image = torch.arange(24, dtype=torch.float64).reshape(1, 2, 3, 4) / 997
mask = torch.arange(6, dtype=torch.bfloat16).reshape(1, 2, 3) / 16
values = {'value_1': 2**63 + 123, 'value_2': False, 'value_3': -0.0, 'value_4': ' 雪\r\n',
'value_5': image.repeat(2, 1, 1, 1), 'value_6': mask.repeat(2, 1, 1),
'value_7': 'master-only unchanged'}
item = {'image': image, 'mask': mask, 'values': values, 'index': 0}
graph = {
'start': node(rt.START_CLASS),
'calculate': node('CalculateOverrides', image=['start', 0], mask=['start', 1],
integer=['start', 4], boolean=['start', 5], floating=['start', 6], text=['start', 7],
reference=['start', 8], reference_mask=['start', 9]),
'end': node(rt.END_CLASS, context=['start', 3], images=['start', 0]),
'preview': node('MustNotExecute', images=['start', 0]),
}
names = ('value_1', 'value_2', 'value_4', 'value_6', 'value_8', 'value_10')
expected = dict(zip(names, (values['value_5'] + image, values['value_6'] + mask,
values['value_4'] + ':calculated', values['value_1'] + 7, -0.0, True)))
graph['end']['inputs'].update({name: ['calculate', index] for index, name in enumerate(names)})
region = {'start_id': 'start', 'end_id': 'end', 'node_ids': set(graph)}
worker = {'id': 'pool', 'type': 'serverless', 'endpoint_id': 'ep', 'timeout_seconds': 30,
'jobs_per_run': 16, 'enabled': True, 'media_profile': 'comfy-media-v1'}
async def run(prompt, request_id):
job = await adapter.prepare_serverless_item(prompt, region, item, worker,
request_id=request_id, job_id='job')
assert not any('Distributed' in n['class_type'] for n in job['workflow'].values())
workflow, media, legacy, targets = handler._validate_request(
{key: job[key] for key in ('workflow', 'images', 'media')})
stage = handler._Staging(folder_paths.get_input_directory(), request_id)
async def submit(native):
assert 'preview' not in native
valid = await execution.validate_prompt(request_id, native, targets)
assert valid[0], valid
executor = execution.PromptExecutor(ps, cache_args={'ram': 0, 'ram_inactive': 0}, asset_manager=assets)
await asyncio.to_thread(executor.execute, native, request_id, {}, targets)
assert executor.success, executor.status_messages
return executor.history_result
async def fetch(artifact):
transport = artifact['transport']
path = Path(folder_paths.get_temp_directory()) / transport['subfolder'] / transport['filename']
return path.read_bytes()
execution_adapter = types.SimpleNamespace(media=media, targets=targets, stage=stage,
remaining=lambda: 30, submit_and_wait=submit, fetch=fetch)
try:
response = await handler._run(execution_adapter, workflow, legacy, None, None)
grouped = await decode(response, media, job['manifest'])
finally:
stage.close(cleanup=True)
result = rt.normalize_worker_result(grouped['job']['images'], values={
name: grouped['job'][name] for name in rt.VALUE_NAMES if name in grouped['job']})
state = rt.ImageListJob('ctx', 'start', 'end', prompt, [item], ['pool'], '')
return job, rt._collect_output_lists([result], state=state, include_values=True)[3][0]
job, actual = await run(graph, 'calculated')
assert set(b['socket'] for b in job['manifest']) == {'images', *names}
assert calls == [values['value_1']]
for name, wanted in {**values, **expected}.items():
got = actual[name]
assert type(got) is type(wanted), name
if isinstance(wanted, torch.Tensor):
assert got.dtype == wanted.dtype and got.shape == wanted.shape
assert torch.equal(got.contiguous().view(torch.uint8), wanted.contiguous().view(torch.uint8)), name
elif isinstance(wanted, float):
assert struct.pack('!d', got) == struct.pack('!d', wanted), name
else:
assert got == wanted, name
# Direct overrides exercise exact VALUE import injection for bool/int/float/
# string, native tensor import slots, and no execution of the calculation.
for name in names:
graph['end']['inputs'].pop(name)
graph['end']['inputs'].update({f'value_{i + 1}': ['start', i + 4] for i in range(6)})
_, direct = await run(graph, 'passthrough')
assert calls == [values['value_1']]
assert type(direct['value_1']) is int and direct['value_1'] == values['value_1']
assert direct['value_2'] is False
assert struct.pack('!d', direct['value_3']) == struct.pack('!d', -0.0)
assert direct['value_4'] == values['value_4']
assert torch.equal(direct['value_5'], values['value_5'])
assert torch.equal(direct['value_6'], values['value_6'])
assert not torch.cuda.is_initialized()
print('IMAGE_LIST_SERVERLESS_VALUES_CPU_OK calculated six types; sparse master overlay; standalone bridge; real PromptExecutor')
if __name__ == '__main__':
asyncio.run(check(bootstrap()))
@@ -0,0 +1,114 @@
"""CPU-only native Comfy execution across private-slot lifetimes."""
import asyncio
import importlib
import os
from pathlib import Path
import sys
import types
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, sys.argv[1])
import comfy.cli_args
comfy.cli_args.args.cpu = True
comfy.cli_args.args.disable_assets = True
import execution
import nodes
pkg = '_local_retention_probe'
for name, path in [(pkg, ROOT), (pkg + '.image_list', ROOT / 'image_list')]:
module = types.ModuleType(name)
module.__path__ = [str(path)]
sys.modules[name] = module
local = importlib.import_module(pkg + '.image_list.local_executor')
CALLS = []
OUTPUTS = []
class Counted:
@classmethod
def INPUT_TYPES(cls):
return {'required': {'value': ('INT',)}}
RETURN_TYPES = ('INT',)
FUNCTION = 'run'
def run(self, value):
if value < 0:
raise ValueError('intentional retention failure')
CALLS.append(value)
return (value,)
class Uncached(Counted):
@classmethod
def IS_CHANGED(cls, **kwargs):
return float('nan')
class Capture:
@classmethod
def INPUT_TYPES(cls):
return {'required': {'value': ('INT',), 'delivery': ('STRING',)}}
RETURN_TYPES = ()
OUTPUT_NODE = True
FUNCTION = 'run'
def run(self, value, delivery):
OUTPUTS.append((value, delivery))
return ()
nodes.NODE_CLASS_MAPPINGS.update(RetentionCounted=Counted, RetentionUncached=Uncached, RetentionCapture=Capture)
async def run(slot, value, delivery, uncached=False):
prompt = {
'source': {'class_type': 'RetentionUncached' if uncached else 'RetentionCounted', 'inputs': {'value': value}},
'end': {'class_type': 'RetentionCapture', 'inputs': {'value': ['source', 0], 'delivery': delivery}},
}
await asyncio.wrap_future(slot.submit(prompt, delivery, 'end'))
async def main():
first = local.LocalExecutionSlot()
await run(first, 1, 'first-1')
await run(first, 2, 'first-2')
await first.close()
assert local._RETAINED_OUTPUTS is not None
second = local.LocalExecutionSlot()
await run(second, 1, 'second-1')
await run(second, 2, 'second-2')
await run(second, 3, 'changed')
await second.close()
assert CALLS == [1, 2, 3], CALLS
assert OUTPUTS == [(1, 'first-1'), (2, 'first-2'), (1, 'second-1'), (2, 'second-2'), (3, 'changed')], OUTPUTS
third = local.LocalExecutionSlot()
await run(third, 4, 'uncached-1', True)
await run(third, 4, 'uncached-2', True)
await third.close()
assert CALLS[-2:] == [4, 4]
comfy.cli_args.args.cache_none = True
disabled = local.LocalExecutionSlot()
await run(disabled, 1, 'disabled-1')
await run(disabled, 1, 'disabled-2')
await disabled.close()
assert CALLS[-2:] == [1, 1]
assert local._RETAINED_OUTPUTS is None
comfy.cli_args.args.cache_none = False
failed = local.LocalExecutionSlot()
await run(failed, 7, 'before-failure')
try:
await run(failed, -1, 'failure')
except RuntimeError:
pass
else:
raise AssertionError('failure fixture unexpectedly succeeded')
await failed.close()
assert local._RETAINED_OUTPUTS is None
cancelled = local.LocalExecutionSlot()
await run(cancelled, 8, 'cancelled')
cancelled.request_stop()
await cancelled.close()
assert local._RETAINED_OUTPUTS is None
oversized = execution.LRUCache(execution.CacheKeySetInputSignature, max_size=local.NATIVE_CACHE_MAX_ENTRIES)
empty = execution.DynamicPrompt({})
await oversized.set_prompt(empty, [], execution.IsChangedCache('bound', empty, oversized))
oversized.min_generation = oversized.generation
oversized.cache = {n: n for n in range(local.NATIVE_CACHE_MAX_ENTRIES + 1)}
local._retain_outputs(oversized)
assert local._RETAINED_OUTPUTS is None
print('PASS: cross-run native cache reuse, fresh delivery, input change, noncacheable nodes, cache_none, cancellation, retention bound')
asyncio.run(main())
+88 -45
View File
@@ -93,11 +93,12 @@ async def check(base):
contract = load("serverless_bridge.contract")
video_codec = load("serverless_bridge.video_codec")
bridge = load("serverless_bridge")
collector = load("nodes.media_collector").DistributedMediaCollector
collector = load("nodes.v3").DistributedCollector
extension = await bridge.comfy_entrypoint()
bridge_classes = await extension.get_node_list()
assert {cls.GET_SCHEMA().node_id for cls in bridge_classes} == {"MediaImportV1", "MediaExportV1"}
sockets = list(contract.SOCKET_KINDS)
collector_sockets = ["images", "audio"]
kinds = list(contract.SOCKET_KINDS.values())
assert kinds == ["IMAGE", "AUDIO", "MASK", "STRING", "VIDEO"]
io_types = [io.Image, io.Audio, io.Mask, io.String, io.Video]
@@ -165,9 +166,10 @@ async def check(base):
assert set(media) == set(expected)
for socket in expected:
assert len(media[socket]) == len(expected[socket]), (tag, socket)
# Collector must preserve native objects and ordered item lists,
# not merge batches, resample, convert types, or encode again.
assert all(a is e for a, e in zip(media[socket], expected[socket])), (tag, socket)
# Existing Collector outputs are a batched IMAGE and one AUDIO,
# not lists. Native V3 wraps each output once for this list sink.
for actual, wanted in zip(media[socket], expected[socket]):
exact(contract.SOCKET_KINDS[socket], actual, wanted)
counts = {socket: len(media[socket]) for socket in expected}
seen_sinks[tag[0]] = counts
return io.NodeOutput(ui={"accepted": [counts]})
@@ -180,6 +182,19 @@ async def check(base):
def node(class_type, **inputs):
return {"class_type": class_type, "inputs": inputs}
def add_bridge_exports(workflow, manifest, request_id, participant, job_id, links):
"""Exercise bridge-only kinds without inventing Collector sockets."""
for socket, link in links.items():
export_id = "bridge-export:" + socket
binding = {"binding_id": "bridge-binding:" + socket, "collector_id": "bridge-only",
"job_id": job_id, "participant_id": participant, "node_id": export_id,
"socket": socket, "kind": contract.SOCKET_KINDS[socket]}
assert export_id not in workflow
manifest.append(binding)
workflow[export_id] = node("MediaExportV1", request_id=request_id,
binding_json=json.dumps(binding), **{socket: link})
contract.validate_bindings(manifest)
def tiny_video(variant):
buffer = bytes_io.BytesIO()
with av.open(buffer, "w", format="matroska") as container:
@@ -199,7 +214,7 @@ async def check(base):
image[0, 0, 0, 0] = -0.0
mask = torch.tensor([[[-0.0, 0.125, 0.875]]], dtype=torch.bfloat16)
audio = {"waveform": torch.arange(38, dtype=torch.float64).reshape(1, 2, 19) / 997,
"sample_rate": 44100 + variant}
"sample_rate": 44100}
audio["waveform"][0, 0, 0] = -0.0
component = VideoFromComponents(VideoComponents(
images=image[..., :3], frame_rate=Fraction(30000, 1001), alpha=mask.reshape(1, 1, 3, 1).repeat(2, 2, 1, 1),
@@ -220,7 +235,7 @@ async def check(base):
with wave.open(str(wav), "wb") as out:
out.setnchannels(2)
out.setsampwidth(2)
out.setframerate(22050)
out.setframerate(44100)
out.writeframes(pcm.tobytes())
waveform, sample_rate = nodes_audio.load(str(wav))
clip = native_root / "native.mkv"
@@ -278,17 +293,22 @@ async def check(base):
participant = "serverless:" + key
request_id = "acceptance-" + key
graph = {"source": node("AcceptanceMediaSource", key=key),
"collector": node("DistributedMediaCollector", load_balance=False,
"collector": node("DistributedCollector", load_balance=False, transfer_format="exact",
multi_job_id="job-main", worker_id=participant,
**{socket: ["source", index] for index, socket in enumerate(sockets)}),
"text-collector": node("DistributedMediaCollector", load_balance=False,
multi_job_id="job-text", worker_id=participant, texts=["source", 3])}
**{socket: ["source", index] for index, socket in enumerate(collector_sockets)}),
"audio-collector": node("DistributedCollector", load_balance=False,
multi_job_id="job-audio", worker_id=participant, audio=["source", 1])}
if key == "worker-a":
graph.update({"wav": node("LoadAudio", audio=wav.name), "clip": node("LoadVideo", file=clip.name)})
graph["collector"]["inputs"].update(audio=["wav", 0], video=["clip", 0])
graph["collector"]["inputs"]["audio"] = ["wav", 0]
before = copy.deepcopy(graph)
workflow, manifest = compiler.compile_media_prompt(graph, request_id)
assert graph == before
bridge_links = {socket: ["source", sockets.index(socket)] for socket in sockets[2:]}
if key == "worker-a":
workflow["clip"] = copy.deepcopy(graph["clip"])
bridge_links["video"] = ["clip", 0]
add_bridge_exports(workflow, manifest, request_id, participant, "job-bridge", bridge_links)
assert len(manifest) == 6 and {b["kind"] for b in manifest} == set(kinds)
folder_paths.set_input_directory(str(native_root))
workflow_before = copy.deepcopy(workflow)
@@ -335,17 +355,19 @@ async def check(base):
# Exercise the wire format, not an in-memory handler-only object.
result = json.loads(json.dumps(result, allow_nan=False))
grouped = await decoder.collect_typed_results(result, media, manifest)
assert set(grouped) == {"job-main", "job-text"}
assert set(grouped["job-main"]) == set(sockets)
assert grouped["job-text"]["texts"] == expectations[key]["texts"]
assert set(grouped) == {"job-main", "job-audio", "job-bridge"}
assert set(grouped["job-main"]) == set(collector_sockets)
assert set(grouped["job-bridge"]) == set(sockets[2:])
exact("AUDIO", grouped["job-audio"]["audio"][0], fixtures[key]["audio"][0])
all_media = {**grouped["job-main"], **grouped["job-bridge"]}
for socket, kind in zip(sockets, kinds):
values, expected = grouped["job-main"][socket], expectations[key][socket]
values, expected = all_media[socket], expectations[key][socket]
assert len(values) == len(expected), (key, socket)
for actual, wanted in zip(values, expected):
exact(kind, actual, wanted)
if key == "worker-b":
original = fixtures[key]["video"][0].get_components()
decoded = grouped["job-main"]["video"][0].get_components()
decoded = grouped["job-bridge"]["video"][0].get_components()
for field in ("images", "alpha"):
assert getattr(decoded, field).data_ptr() != getattr(original, field).data_ptr()
assert decoded.audio["waveform"].data_ptr() != original.audio["waveform"].data_ptr()
@@ -363,7 +385,7 @@ async def check(base):
assert ps.distributed_pending_jobs == {}
# Exercise user-selected image return encoding through the actual
# compiler, HTTP handler, V3 exporter and atomic response decoder.
for fmt in ("png", "jpeg", "h264"):
for fmt in ("legacy_png", "png", "jpeg", "h264"):
key = "encoded-" + fmt
fixtures[key] = native_fixtures(3)
# Keep full alpha precision and use nontrivial image dimensions.
@@ -372,12 +394,23 @@ async def check(base):
fixtures[key]["images"] = [rgba]
request_id = "request-" + key
graph = {"source": node("AcceptanceMediaSource", key=key),
"collector": node("DistributedMediaCollector", transfer_format=fmt,
"collector": node("DistributedCollector", transfer_format=fmt,
transfer_quality=73, multi_job_id="job-" + key,
worker_id="serverless:" + key,
**{socket: ["source", i] for i, socket in enumerate(sockets)})}
**{socket: ["source", i] for i, socket in enumerate(collector_sockets)})}
if fmt == "legacy_png":
# A saved Collector with no newly selected controls still uses
# legacy_png / 90, mapped to PNG by the serverless compiler.
graph["collector"]["inputs"].pop("transfer_format")
graph["collector"]["inputs"].pop("transfer_quality")
original = copy.deepcopy(graph)
workflow, manifest = compiler.compile_media_prompt(graph, request_id)
if fmt == "legacy_png":
binding = next(b for b in manifest if b["kind"] == "IMAGE")
controls = workflow[binding["node_id"]]["inputs"]
assert (controls["transfer_format"], controls["transfer_quality"]) == ("png", 90)
add_bridge_exports(workflow, manifest, request_id, "serverless:" + key, "job-bridge-" + key,
{socket: ["source", sockets.index(socket)] for socket in sockets[2:]})
media = {"version": 1, "profile": contract.PROFILE, "request_id": request_id,
"artifacts": [], "inputs": [], "outputs": manifest}
body = json.loads(serializer(workflow, [], 30, media=media))
@@ -385,23 +418,32 @@ async def check(base):
input_dir=base / "input", temp_dir=base / "temp", timeout_seconds=30)
assert graph == original
grouped = await decoder.collect_typed_results(result, media, manifest)
actual = grouped["job-" + key]
actual = {**grouped["job-" + key], **grouped["job-bridge-" + key]}
image = actual["images"][0]
assert image.shape == rgba.shape and image.dtype == rgba.dtype
assert torch.equal(image[..., 3], rgba[..., 3]), "alpha must not be discarded/narrowed"
assert not torch.equal(image[..., :3], rgba[..., :3]), "selection must actually encode"
if fmt == "png":
if fmt in ("legacy_png", "png"):
pixels = (rgba[..., :3].float().clamp(0, 1) * 255).to(torch.uint8)
assert torch.equal(image[..., :3], (pixels.float() / 255).to(rgba.dtype))
for socket, kind in zip(sockets[1:], kinds[1:]):
for got, wanted in zip(actual[socket], fixtures[key][socket]):
exact(kind, got, video_codec.encode_video(wanted) if kind == "VIDEO" else wanted)
assert not list((base / "temp").rglob("*.bin"))
print("IMAGE_TRANSFER_HTTP_OK png jpeg h264; alpha and other media exact")
print("REAL_HTTP_BRIDGE_RESULTS_OK all five kinds; 6 bindings/request; native VIDEO remains readable after cleanup")
print("IMAGE_TRANSFER_HTTP_OK default legacy_png -> png/90; png jpeg h264; alpha and other media exact")
print("REAL_HTTP_BRIDGE_RESULTS_OK Collector IMAGE/AUDIO + independent MASK/STRING/VIDEO bridge exports; native VIDEO survives cleanup")
print("ATOMIC_DECODE_REJECTION_OK corrupt final artifact rejected before delivery")
participants = list(grouped_results)
def expected_batches(ordered, include_images=True):
audio_items = [item for media in ordered for item in media["audio"]]
expected = {"audio": [{"waveform": torch.cat([item["waveform"] for item in audio_items], dim=-1),
"sample_rate": audio_items[0]["sample_rate"]}]}
if include_images:
expected["images"] = [torch.cat([image for media in ordered for image in media["images"]])]
return expected
for delegate in (False, True):
tag = "delegate" if delegate else "master"
job = "collect-" + tag
@@ -410,39 +452,40 @@ async def check(base):
await queue.put({"worker_id": participant, "media": grouped_results[participant]["job-main"], "is_last": True})
ps.distributed_pending_jobs[job] = queue
ordered = ([] if delegate else [fixtures["local"]]) + [grouped_results[p]["job-main"] for p in participants]
expected_sinks[tag] = {socket: [item for values in ordered for item in values[socket]] for socket in sockets}
graph = {"collector": node("DistributedMediaCollector", load_balance=False, multi_job_id=job,
enabled_worker_ids=json.dumps(participants), expected_media=json.dumps(sockets), delegate_only=delegate),
expected_sinks[tag] = expected_batches(ordered)
graph = {"collector": node("DistributedCollector", load_balance=False, multi_job_id=job,
enabled_worker_ids=json.dumps(participants), delegate_only=delegate, transfer_format="exact"),
"assert": node("AcceptanceMediaAssert", tag=tag,
**{socket: ["collector", index] for index, socket in enumerate(sockets)})}
**{socket: ["collector", index] for index, socket in enumerate(collector_sockets)})}
if not delegate:
graph["source"] = node("AcceptanceMediaSource", key="local")
graph["collector"]["inputs"].update({socket: ["source", index] for index, socket in enumerate(sockets)})
# The second group from the very same complete response must reach its
# own Collector, rather than being folded into the first collector job.
text_job, text_tag = job + "-text", tag + "-text"
text_queue = asyncio.Queue()
graph["collector"]["inputs"].update({socket: ["source", index] for index, socket in enumerate(collector_sockets)})
# A second audio-only Collector consumes its own job group from the same
# atomic response. It must return None for images and batch audio in order.
audio_job, audio_tag = job + "-audio", tag + "-audio"
audio_queue = asyncio.Queue()
for participant in reversed(participants):
await text_queue.put({"worker_id": participant,
"media": grouped_results[participant]["job-text"], "is_last": True})
ps.distributed_pending_jobs[text_job] = text_queue
text_ordered = ([] if delegate else [fixtures["local"]]) + [grouped_results[p]["job-text"] for p in participants]
expected_sinks[text_tag] = {"texts": [item for values in text_ordered for item in values["texts"]]}
graph["text-collector"] = node("DistributedMediaCollector", load_balance=False, multi_job_id=text_job,
enabled_worker_ids=json.dumps(participants), expected_media='["texts"]', delegate_only=delegate)
graph["text-assert"] = node("AcceptanceMediaAssert", tag=text_tag, texts=["text-collector", 3])
await audio_queue.put({"worker_id": participant,
"media": grouped_results[participant]["job-audio"], "is_last": True})
ps.distributed_pending_jobs[audio_job] = audio_queue
audio_ordered = ([] if delegate else [fixtures["local"]]) + [grouped_results[p]["job-audio"] for p in participants]
expected_sinks[audio_tag] = expected_batches(audio_ordered, include_images=False)
graph["audio-collector"] = node("DistributedCollector", load_balance=False, multi_job_id=audio_job,
enabled_worker_ids=json.dumps(participants), delegate_only=delegate)
graph["audio-assert"] = node("AcceptanceMediaAssert", tag=audio_tag, audio=["audio-collector", 1])
if not delegate:
graph["text-collector"]["inputs"]["texts"] = ["source", 3]
graph["audio-collector"]["inputs"]["audio"] = ["source", 1]
valid = await execution.validate_prompt(job, graph, None)
assert valid[0] and not valid[3], valid
executor = execution.PromptExecutor(ps, cache_args={"ram": 0, "ram_inactive": 0}, asset_manager=asset_manager)
await asyncio.to_thread(executor.execute, graph, job, {}, valid[2])
assert executor.success, executor.status_messages
assert executor.history_result["outputs"]["assert"]["accepted"] == [seen_sinks[tag]]
assert executor.history_result["outputs"]["text-assert"]["accepted"] == [seen_sinks[text_tag]]
assert job not in ps.distributed_pending_jobs and text_job not in ps.distributed_pending_jobs
assert source_calls == ["worker-a", "worker-b", "encoded-png", "encoded-jpeg", "encoded-h264", "local"], source_calls
assert set(seen_sinks) == {"master", "delegate", "master-text", "delegate-text"}
assert executor.history_result["outputs"]["audio-assert"]["accepted"] == [seen_sinks[audio_tag]]
assert job not in ps.distributed_pending_jobs and audio_job not in ps.distributed_pending_jobs
assert source_calls == ["worker-a", "worker-b", "encoded-legacy_png", "encoded-png", "encoded-jpeg", "encoded-h264", "local"], source_calls
assert set(seen_sinks) == {"master", "delegate", "master-audio", "delegate-audio"}
assert not list((base / "output").rglob("*"))
assert not torch.cuda.is_initialized()
print("MASTER_AND_DELEGATE_ORDER_OK", json.dumps(seen_sinks, sort_keys=True))
+48 -9
View File
@@ -36,7 +36,7 @@ BASELINE = json.loads((ROOT / 'tests/fixtures/v1_node_contracts.json').read_text
ADDED_OPTIONAL_INPUTS = {
'DistributedCollector': {
'transfer_format': [
['legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1'],
['legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1', 'exact'],
{'default': 'legacy_png'},
],
'transfer_quality': ['INT', {'default': 90, 'min': 1, 'max': 100, 'step': 1}],
@@ -45,7 +45,8 @@ ADDED_OPTIONAL_INPUTS = {
# Nodes added after the V1 baseline; not part of saved-workflow contracts.
# DistributedWorkerSink is internal (dev-only) and only inserted by the master.
ADDED_NODES = {'DistributedWorkerSink', 'DistributedImageListStart', 'DistributedImageListEnd', 'DistributedMediaCollector'}
ADDED_NODES = {'DistributedWorkerSink', 'DistributedImageListStart', 'DistributedImageListEnd',
'DistributedImageListGetValue'}
def normalized_input(value):
@@ -273,7 +274,7 @@ async def check_image_list_lazy_executor(mapping, module, prompt_server, asset_m
original_contexts = set(runtime._MASTER_JOBS)
image_list_nodes = sys.modules[module.__name__ + '.nodes.image_list']
workflow_meta = {'nodes': [{'id': 2, 'type': 'DistributedImageListStart'}], 'extra': {'acceptance': True}}
async def fake_worker_results(_context_id, *, model_callback=None):
async def fake_worker_results(_context_id, *, model_callback=None, include_values=False):
state = runtime.get_master_job(_context_id)
assert state.prompt_id == 'image-list-lazy', state.prompt_id
assert state.workflow_meta == workflow_meta, state.workflow_meta
@@ -281,6 +282,7 @@ async def check_image_list_lazy_executor(mapping, module, prompt_server, asset_m
return (
[torch.full((1, 8, 8, 3), value, dtype=torch.float32) for value in (0.125, 0.5, 0.875)],
[None, None, None], [None, None, None],
[{}, {}, {}],
)
graph = {
@@ -366,14 +368,16 @@ async def main():
extension = await module.comfy_entrypoint()
classes = await extension.get_node_list()
mapping = {cls.GET_SCHEMA().node_id: cls for cls in classes}
assert len(classes) == len(mapping) == len(BASELINE) + len(ADDED_NODES)
assert len(classes) == len(mapping) == len(BASELINE) + len(ADDED_NODES) == 12
assert set(mapping) == set(BASELINE) | ADDED_NODES
for node_id in ADDED_NODES:
assert nodes.NODE_CLASS_MAPPINGS[node_id] is mapping[node_id]
media_info = mapping['DistributedMediaCollector'].GET_NODE_INFO_V1()
assert media_info['output'] == ['IMAGE', 'AUDIO', 'MASK', 'STRING', 'VIDEO'], media_info
assert all(media_info['output_is_list']), media_info
assert media_info['input']['optional']['texts'][1]['forceInput'] is True
assert 'DistributedMediaCollector' not in mapping
collector_info = mapping['DistributedCollector'].GET_NODE_INFO_V1()
assert collector_info['output'] == ['IMAGE', 'AUDIO'], collector_info
assert not any(collector_info['output_is_list']), collector_info
assert list(collector_info['input']['optional']) == [
'images', 'audio', 'transfer_format', 'transfer_quality']
sink_info = mapping['DistributedWorkerSink'].GET_NODE_INFO_V1()
assert sink_info['output_node'] and sink_info['dev_only'], sink_info
for node_id, cls in mapping.items():
@@ -427,6 +431,42 @@ async def main():
assert start_schema.display_name == 'Distributed Image List Start'
assert end_schema.node_id == 'DistributedImageListEnd'
assert end_schema.display_name == 'Distributed Image List End'
runtime = sys.modules[module.__name__ + '.image_list.runtime']
assert [input.id for input in end_schema.inputs] == [
'images', 'masks', 'texts', 'context', 'transfer_format', 'transfer_quality'] + list(runtime.VALUE_NAMES)
format_input, quality_input = end_schema.inputs[4:6]
assert format_input.options == list(runtime.IMAGE_TRANSFER_FORMATS)
assert format_input.default == 'exact' and quality_input.default == 90
assert format_input.optional and quality_input.optional
assert [output.io_type for output in end_schema.outputs] == [
'IMAGE', 'MASK', 'STRING', 'DISTRIBUTED_IMAGE_LIST_VALUES']
assert end_schema.outputs[3].id == end_schema.outputs[3].display_name == 'values'
assert all(output.is_output_list for output in end_schema.outputs)
for override in end_schema.inputs[6:]:
assert override.optional and override.lazy
assert override.get_io_type() == 'IMAGE,MASK,STRING,INT,FLOAT,BOOLEAN'
getter = mapping['DistributedImageListGetValue']
getter_schema = getter.GET_SCHEMA()
assert getter_schema.display_name == 'Distributed Image List Get Value'
assert not getter_schema.is_input_list
assert getter_schema.outputs[0].id == getter_schema.outputs[0].display_name == 'value'
assert getter_schema.outputs[0].io_type == '*'
assert getter_schema.inputs[1].options == list(runtime.VALUE_NAMES)
assert getter_schema.inputs[1].default == 'value_1'
# Real core maps scalar mappings, never arbitrary tensor batches.
tensor = torch.ones(3, 2, 2, 3)
data, missing, hidden = execution.get_input_data({'value_name': 'value_1'}, getter, 'get-value')
data['values'] = [{'value_1': tensor}, {'value_1': 17}, {'value_1': False}]
mapped = await execution._async_map_node_over_list('get-value', 'get-value', getter, data,
getter.FUNCTION, v3_data=hidden)
mapped = await execution.resolve_map_node_over_list_results(mapped)
assert len(mapped) == 3 and mapped[0].result[0] is tensor
assert mapped[1].result == (17,) and mapped[2].result == (False,)
try:
getter.execute({}, 'value_1')
raise AssertionError('missing Get Value key accepted')
except ValueError as exc:
assert 'absent' in str(exc)
texts_input = next(input for input in end_schema.inputs if input.id == 'texts')
texts_schema = texts_input.as_dict()
assert texts_schema.get('forceInput') is True, texts_schema
@@ -470,7 +510,6 @@ async def main():
rows = runtime_mod.normalize_image_list_inputs(prepared['images'], None, seen['values'])
assert [row['values'] for row in rows] == [{'value_1': 'shared', 'value_3': n} for n in (4, 5, 6)]
print('IMAGE_LIST_AUTOGROW_OK real parser; value_N slots shared/per-item')
assert [out.io_type for out in end_schema.outputs] == ['IMAGE', 'MASK', 'STRING']
assert all(out.is_output_list for out in end_schema.outputs)
assert end_schema.is_input_list and end_schema.is_output_node
print('IMAGE_LIST_SCHEMA_OK exact IDs, names, typed list outputs')
+524
View File
@@ -0,0 +1,524 @@
"""Exact Collector transport through real serializers, HTTP, route and queue.
Only ComfyUI's process-wide server/model-management and unrelated orchestration
are replaced. Tensor/audio codecs, multipart bodies and aiohttp are real; no
running ComfyUI service, GPU, provider or external network access is required.
"""
import asyncio
from contextlib import asynccontextmanager
import importlib
import importlib.util
import json
from pathlib import Path
import sys
import types
from unittest.mock import AsyncMock
import pytest
import torch
ROOT = Path(__file__).resolve().parents[1]
PKG = "dist_collector_exact_testpkg"
JOB_ID = "exact-job"
WORKER_ID = "worker-1"
@pytest.fixture
def transport(monkeypatch):
# Other legacy test loaders may leave aiohttp stubs behind. Replace only
# those entries, never unload/reload real extension-backed dependencies.
for name, loaded_module in list(sys.modules.items()):
if (name == "aiohttp" or name.startswith("aiohttp.")) and not getattr(loaded_module, "__file__", None):
monkeypatch.delitem(sys.modules, name)
aiohttp = importlib.import_module("aiohttp")
web = importlib.import_module("aiohttp.web")
def module(name, **attrs):
result = types.ModuleType(name)
result.__dict__.update(attrs)
monkeypatch.setitem(sys.modules, name, result)
return result
def load(relative):
name = f"{PKG}.{relative}"
spec = importlib.util.spec_from_file_location(name, ROOT.joinpath(*relative.split(".")).with_suffix(".py"))
assert spec is not None and spec.loader is not None
result = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, name, result)
spec.loader.exec_module(result)
parent, _, child = name.rpartition(".")
setattr(sys.modules[parent], child, result)
return result
module(PKG, __path__=[str(ROOT)])
for package in ("nodes", "api", "utils", "serverless_bridge"):
module(f"{PKG}.{package}", __path__=[str(ROOT / package)])
server = types.SimpleNamespace(routes=web.RouteTableDef(), distributed_pending_jobs={})
module("server", PromptServer=types.SimpleNamespace(instance=server))
mm = module("comfy.model_management", throw_exception_if_processing_interrupted=lambda: None)
progress = module("comfy.utils", ProgressBar=lambda _total: types.SimpleNamespace(update=lambda _n: None))
module("comfy", __path__=[], model_management=mm, utils=progress)
module(f"{PKG}.utils.logging", debug_log=lambda *_a, **_kw: None, log=lambda *_a, **_kw: None)
module(f"{PKG}.utils.config", get_worker_timeout_seconds=lambda: 0.1,
load_config=lambda: {"workers": []}, is_master_delegate_only=lambda: False)
async def handle_api_error(_request, error, status=500):
return web.json_response({"error": str(error)}, status=status)
module(f"{PKG}.utils.network", handle_api_error=handle_api_error,
get_client_session=AsyncMock(), build_worker_url=lambda _worker: "http://unused.invalid",
probe_worker=AsyncMock(return_value=None))
module(f"{PKG}.utils.async_helpers", run_async_in_server_loop=asyncio.run)
module(f"{PKG}.api.queue_orchestration", ensure_distributed_state=lambda: None,
orchestrate_distributed_execution=AsyncMock())
module(f"{PKG}.api.queue_request", parse_queue_request_payload=lambda value: value)
for name in ("utils.constants", "utils.image", "utils.audio_payload", "utils.transfer_codec"):
load(name)
bridge_codec = load("serverless_bridge.codec")
load("serverless_bridge.image_transfer")
image_codec = load("serverless_bridge.image_codec")
collector = load("nodes.collector")
routes = load("api.job_routes")
return types.SimpleNamespace(aiohttp=aiohttp, web=web, server=server,
collector=collector, routes=routes, bridge_codec=bridge_codec,
image_codec=image_codec)
@asynccontextmanager
async def endpoint(transport, monkeypatch, *, request_limit=2 * 1024 * 1024,
chunk_target=1024 * 1024, client_max_size=None, handler=None):
"""A disposable loopback endpoint; record actual HTTP lengths, not estimates."""
queue = asyncio.Queue()
transport.server.distributed_jobs_lock = asyncio.Lock()
transport.server.distributed_pending_jobs = {JOB_ID: queue}
requests = []
web = transport.web
@web.middleware
async def record_request(request, handler):
record = {"bytes": request.content_length, "content_type": request.content_type}
requests.append(record)
response = await handler(request)
record["status"] = response.status
if response.status < 400 and request.content_type.startswith("multipart/"):
form = await request.post()
record["envelope"] = json.loads(form["envelope"])
return response
app = web.Application(client_max_size=client_max_size or request_limit, middlewares=[record_request])
app.router.add_post("/distributed/job_complete", handler or transport.routes.job_complete_endpoint)
runner = web.AppRunner(app)
await runner.setup()
try:
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
port = site._server.sockets[0].getsockname()[1]
async with transport.aiohttp.ClientSession() as session:
monkeypatch.setattr(transport.collector, "get_client_session", AsyncMock(return_value=session))
monkeypatch.setattr(transport.collector, "TRANSFER_MAX_REQUEST_BYTES", request_limit)
monkeypatch.setattr(transport.collector, "TRANSFER_MAX_CHUNK_BYTES", chunk_target)
yield types.SimpleNamespace(
session=session, url=f"http://127.0.0.1:{port}", requests=requests, queue=queue,
node=transport.collector.DistributedCollectorNode(),
)
finally:
await runner.cleanup()
def frames(dtype=torch.float32, channels=3, count=5, height=7, width=9):
# Non-contiguous, out-of-range and non-8-bit-representable values exercise
# detach/contiguous without permitting clipping, alpha loss or quantization.
values = torch.arange(count * height * width * channels, dtype=torch.float64)
values = (values.remainder(41) - 11) / 17
images = values.reshape(count, width, height, channels).transpose(1, 2).to(dtype)
images[0, 0, 0, 0] = -0.0
return images.requires_grad_(True)
def assert_exact(actual, expected):
assert actual.dtype == expected.dtype
assert actual.shape == expected.shape
assert actual.device.type == "cpu"
assert actual.is_contiguous()
# Compare raw bits as well as values, including BF16 and signed zero.
assert torch.equal(actual, expected)
assert torch.equal(actual.view(torch.uint8), expected.detach().contiguous().view(torch.uint8))
def queued_items(queue):
return [queue.get_nowait() for _ in range(queue.qsize())]
async def send(ctx, images, *, audio=None, fmt="exact"):
await ctx.node.send_batch_to_master(images, audio, JOB_ID, ctx.url, WORKER_ID, fmt, 17)
@pytest.mark.parametrize("delegate_only", [False, True])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32, torch.float64])
@pytest.mark.parametrize("channels", [1, 3, 4])
def test_exact_sender_route_and_collector_preserve_dtype_values_and_channels(
transport, monkeypatch, dtype, channels, delegate_only):
images = frames(dtype, channels)
frame_bytes = images[0].numel() * images.element_size()
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
async def scenario():
async with endpoint(transport, monkeypatch, request_limit=reserve + 2 * frame_bytes,
chunk_target=2 * frame_bytes) as ctx:
await send(ctx, images)
items = queued_items(ctx.queue)
assert [item["image_index"] for item in items] == list(range(len(images)))
assert [item["is_last"] for item in items] == [False] * 4 + [True]
assert all(item["worker_id"] == WORKER_ID and item["format"] == "exact" for item in items)
for item, expected in zip(items, images.split(1)):
assert_exact(item["tensor"], expected)
await ctx.queue.put(item)
envelopes = [record["envelope"] for record in ctx.requests]
assert [(e["start_index"], e["count"]) for e in envelopes] == [(0, 2), (2, 2), (4, 1)]
assert all(record["bytes"] <= reserve + 2 * frame_bytes for record in ctx.requests)
master_images = None if delegate_only else images[:1].detach().clone().add(0.03125)
actual, _audio = await ctx.node.execute(
master_images, None, multi_job_id=JOB_ID, enabled_worker_ids=json.dumps([WORKER_ID]),
delegate_only=delegate_only, transfer_format="exact",
)
expected = images if delegate_only else torch.cat([master_images, images])
assert_exact(actual, expected)
assert JOB_ID not in transport.server.distributed_pending_jobs
asyncio.run(scenario())
def test_exact_single_frame_can_exceed_chunk_target_but_not_request_limit(transport, monkeypatch):
images = frames(count=3)
frame_bytes = images[0].numel() * images.element_size()
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
async def scenario():
async with endpoint(transport, monkeypatch, request_limit=reserve + frame_bytes,
chunk_target=frame_bytes // 2) as ctx:
await send(ctx, images)
assert [record["envelope"]["count"] for record in ctx.requests] == [1, 1, 1]
assert all(record["bytes"] <= reserve + frame_bytes for record in ctx.requests)
assert_exact(torch.cat([item["tensor"] for item in queued_items(ctx.queue)]), images)
asyncio.run(scenario())
def test_exact_oversized_frame_is_rejected_before_encoding_or_post(transport, monkeypatch):
images = frames(count=3)
frame_bytes = images[0].numel() * images.element_size()
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
encode = AsyncMock(side_effect=AssertionError("oversized frame must not be encoded"))
monkeypatch.setattr(transport.collector, "_encode_exact_frames", encode)
async def scenario():
async with endpoint(transport, monkeypatch, request_limit=reserve + frame_bytes - 1) as ctx:
with pytest.raises(transport.collector.transfer_codec.FrameTooLargeError, match="request limit"):
await send(ctx, images)
assert ctx.requests == []
assert ctx.queue.empty()
encode.assert_not_called()
asyncio.run(scenario())
@pytest.mark.parametrize("shape", [(0, 2, 3, 3), (1, 0, 3, 3), (1, 2, 0, 3), (1, 2, 3, 0),
(1, 2, 3, 2), (1, 2, 3, 5), (1, 2, 3), (2, 3)])
def test_exact_sender_rejects_invalid_geometry_before_post(transport, monkeypatch, shape):
image = torch.empty(shape)
with pytest.raises(transport.collector.transfer_codec.TransferCodecError):
transport.collector._exact_frame_bytes(image)
async def scenario():
async with endpoint(transport, monkeypatch) as ctx:
# An empty batch is also a no-media completion error (ValueError).
with pytest.raises(ValueError):
await send(ctx, image)
assert ctx.requests == []
assert ctx.queue.empty()
asyncio.run(scenario())
@pytest.mark.parametrize("dtype", [torch.uint8, torch.int32, torch.bool, torch.complex64])
def test_exact_sender_rejects_unsupported_dtype_before_post(transport, monkeypatch, dtype):
async def scenario():
async with endpoint(transport, monkeypatch) as ctx:
with pytest.raises(transport.collector.transfer_codec.TransferCodecError, match="dtype"):
await send(ctx, torch.zeros(1, 2, 3, 3, dtype=dtype))
assert ctx.requests == []
asyncio.run(scenario())
def test_exact_frame_planning_enforces_bridge_byte_limit(transport, monkeypatch):
monkeypatch.setattr(transport.bridge_codec, "MAX_PAYLOAD_BYTES", 32)
with pytest.raises(transport.collector.transfer_codec.TransferCodecError, match="[Bb]yte|limit"):
transport.collector._exact_frame_bytes(torch.zeros(1, 2, 3, 3))
def test_exact_frame_planning_rejects_non_dense_layout(transport):
image = torch.zeros(1, 2, 3, 3).to_sparse()
with pytest.raises(transport.collector.transfer_codec.TransferCodecError, match="dense|layout|strided"):
transport.collector._exact_frame_bytes(image)
@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")])
def test_exact_sender_rejects_nonfinite_payload(transport, monkeypatch, value):
images = torch.full((1, 2, 3, 3), value)
async def scenario():
async with endpoint(transport, monkeypatch) as ctx:
with pytest.raises(transport.collector.transfer_codec.TransferCodecError, match="finite"):
await send(ctx, images)
assert ctx.requests == []
assert ctx.queue.empty()
asyncio.run(scenario())
def exact_chunk(transport, count=2):
images = frames(count=count).detach()
encoded = [transport.bridge_codec.encode_payload("IMAGE", frame) for frame in images.split(1)]
return {
"job_id": JOB_ID, "worker_id": WORKER_ID, "format": "exact",
"start_index": 0, "count": count, "width": images.shape[2], "height": images.shape[1],
"is_last": True, "descriptors": [descriptor for descriptor, _ in encoded],
}, [payload for _, payload in encoded]
@pytest.mark.parametrize("case", [
"zero_count", "bool_width", "negative_height", "negative_index", "bool_index",
"missing_descriptor", "extra_descriptor", "missing_part", "extra_part", "not_a_descriptor",
"wrong_height", "wrong_batch", "bad_channels", "bad_dtype", "bad_layout", "bad_encoding",
"extra_descriptor_field", "truncated", "trailing_bytes", "nonfinite", "compressed_spoof",
])
def test_exact_route_rejects_bad_geometry_and_payload_atomically(transport, monkeypatch, case):
envelope, parts = exact_chunk(transport)
descriptor = envelope["descriptors"][-1]
if case == "zero_count":
envelope["count"] = 0
elif case == "bool_width":
envelope["width"] = True
elif case == "negative_height":
envelope["height"] = -1
elif case == "negative_index":
envelope["start_index"] = -1
elif case == "bool_index":
envelope["start_index"] = True
elif case == "missing_descriptor":
envelope["descriptors"].pop()
elif case == "extra_descriptor":
envelope["descriptors"].append(dict(descriptor))
elif case == "missing_part":
parts.pop()
elif case == "extra_part":
parts.append(parts[-1])
elif case == "not_a_descriptor":
envelope["descriptors"][-1] = None
elif case == "wrong_height":
descriptor["shape"][1] += 1
elif case == "wrong_batch":
descriptor["shape"][0] = 2
elif case == "bad_channels":
descriptor["shape"][-1] = 2
elif case == "bad_dtype":
descriptor["dtype"] = "uint8"
elif case == "bad_layout":
descriptor["layout"] = "BCHW"
elif case == "bad_encoding":
descriptor["encoding"] = "pickle"
elif case == "extra_descriptor_field":
descriptor["unknown"] = True
elif case == "truncated":
parts[-1] = parts[-1][:-1]
elif case == "trailing_bytes":
parts[-1] += b"\0"
elif case == "nonfinite":
bad = torch.full(tuple(descriptor["shape"]), float("nan"))
parts[-1] = bad.numpy().tobytes()
elif case == "compressed_spoof":
# A valid serverless image-transfer descriptor must NOT be accepted as
# exact by the direct route, even though the shared bridge can decode it.
envelope["descriptors"][-1], parts[-1] = transport.image_codec.encode_image(
frames(count=1).detach(), transfer_format="png", transfer_quality=90,
)
# Prove the spoof is a valid compressed bridge payload, not simply junk.
decoded = transport.bridge_codec.decode_payload("IMAGE", envelope["descriptors"][-1], parts[-1])
assert list(decoded.shape) == descriptor["shape"]
async def scenario():
async with endpoint(transport, monkeypatch) as ctx:
async with ctx.session.post(
ctx.url + "/distributed/job_complete",
data=ctx.node._build_multipart(envelope, parts),
) as response:
body = await response.text()
assert response.status == 400, body
# A corrupt second frame must not leave the first in the queue.
assert ctx.queue.empty()
asyncio.run(scenario())
@pytest.mark.parametrize("status,body", [(400, "Invalid JSON payload: old master"),
(413, "upload limit"), (500, "unexpected failure")])
def test_exact_master_rejection_never_falls_back(transport, monkeypatch, status, body):
async def reject(_request):
return transport.web.Response(status=status, text=body)
async def scenario():
async with endpoint(transport, monkeypatch, handler=reject, chunk_target=1) as ctx:
legacy = AsyncMock(side_effect=AssertionError("exact must never use legacy transport"))
compressed = AsyncMock(side_effect=AssertionError("exact must never use compressed transport"))
monkeypatch.setattr(ctx.node, "_send_legacy_batch", legacy)
monkeypatch.setattr(ctx.node, "_send_compressed_batch", compressed)
with pytest.raises(RuntimeError, match=f"HTTP {status}") as raised:
await send(ctx, frames(count=3))
assert "never" in str(raised.value) and "8-bit" in str(raised.value)
assert len(ctx.requests) == 1
assert ctx.requests[0]["content_type"].startswith("multipart/")
assert ctx.queue.empty()
legacy.assert_not_called()
compressed.assert_not_called()
asyncio.run(scenario())
@pytest.mark.parametrize("fmt", ["legacy_png", "png"])
def test_exact_collector_rejects_old_worker_8bit_result(transport, monkeypatch, fmt):
async def scenario():
async with endpoint(transport, monkeypatch) as ctx:
await send(ctx, frames(count=1).detach(), fmt=fmt)
assert ctx.queue.qsize() == 1
with pytest.raises(RuntimeError, match="requested exact"):
await ctx.node.execute(
None, None, multi_job_id=JOB_ID, enabled_worker_ids=json.dumps([WORKER_ID]),
delegate_only=True, transfer_format="exact",
)
assert JOB_ID not in transport.server.distributed_pending_jobs
asyncio.run(scenario())
def test_exact_collector_shape_mismatch_raises_instead_of_returning_local_images(transport, monkeypatch):
async def scenario():
async with endpoint(transport, monkeypatch) as ctx:
await send(ctx, frames(count=1, channels=4))
with pytest.raises(RuntimeError, match="shape|combin|mismatch"):
await ctx.node.execute(
frames(count=1, channels=3), None, multi_job_id=JOB_ID,
enabled_worker_ids=json.dumps([WORKER_ID]), transfer_format="exact",
)
assert JOB_ID not in transport.server.distributed_pending_jobs
asyncio.run(scenario())
def test_exact_incomplete_worker_raises_and_cleans_queue(transport, monkeypatch):
"""A timed-out worker's first frame is not a successful partial result."""
envelope, parts = exact_chunk(transport, count=1)
envelope["is_last"] = False
async def scenario():
async with endpoint(transport, monkeypatch) as ctx:
async with ctx.session.post(
ctx.url + "/distributed/job_complete", data=ctx.node._build_multipart(envelope, parts),
) as response:
assert response.status == 200, await response.text()
assert ctx.queue.qsize() == 1
with pytest.raises(RuntimeError, match="Exact transfer incomplete"):
await asyncio.wait_for(ctx.node.execute(
frames(count=1).detach(), None, multi_job_id=JOB_ID,
enabled_worker_ids=json.dumps([WORKER_ID]), transfer_format="exact",
), timeout=2)
assert JOB_ID not in transport.server.distributed_pending_jobs
asyncio.run(scenario())
def test_exact_oversized_multipart_metadata_is_rejected_before_post(transport, monkeypatch):
images = frames(count=1)
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
limit = reserve + images.numel() * images.element_size()
async def scenario():
async with endpoint(transport, monkeypatch, request_limit=limit) as ctx:
with pytest.raises(transport.collector.transfer_codec.FrameTooLargeError, match="request limit"):
await ctx.node.send_batch_to_master(
images, None, JOB_ID, ctx.url, "w" * limit, "exact", 90,
)
assert ctx.requests == []
assert ctx.queue.empty()
asyncio.run(scenario())
def test_exact_actual_server_upload_rejection_is_413_without_retry(transport, monkeypatch):
async def scenario():
async with endpoint(transport, monkeypatch, client_max_size=1024) as ctx:
with pytest.raises(RuntimeError, match="HTTP 413"):
await send(ctx, frames(count=3, height=64, width=64))
assert len(ctx.requests) == 1
assert ctx.requests[0]["status"] == 413
assert ctx.queue.empty()
asyncio.run(scenario())
def test_exact_oversized_audio_is_rejected_before_any_image_post(transport, monkeypatch):
images = frames(count=3)
audio = {"waveform": torch.zeros(1, 1, 100000), "sample_rate": 48000}
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
async def scenario():
async with endpoint(transport, monkeypatch, request_limit=reserve + 1024) as ctx:
with pytest.raises(transport.collector.transfer_codec.FrameTooLargeError, match="audio.*request limit"):
await send(ctx, images, audio=audio)
assert ctx.requests == []
assert ctx.queue.empty()
asyncio.run(scenario())
def test_exact_tiny_frames_cannot_overflow_multipart_metadata_budget(transport, monkeypatch):
images = torch.full((2048, 1, 1, 1), 0.125, dtype=torch.float16)
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
limit = reserve + images.numel() * images.element_size()
async def scenario():
async with endpoint(transport, monkeypatch, request_limit=limit, chunk_target=limit) as ctx:
await send(ctx, images)
assert len(ctx.requests) > 1
assert all(record["bytes"] <= limit for record in ctx.requests)
assert_exact(torch.cat([item["tensor"] for item in queued_items(ctx.queue)]), images)
asyncio.run(scenario())
@pytest.mark.parametrize("separate", [False, True])
def test_exact_audio_is_preserved_and_only_final_completion_is_terminal(transport, monkeypatch, separate):
images = frames(count=3)
audio = {"waveform": torch.linspace(-0.9, 0.9, 60).reshape(1, 1, -1), "sample_rate": 48000}
frame_bytes = images[0].numel() * images.element_size()
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
media_budget = frame_bytes if separate else frame_bytes + 8192
limit = reserve + media_budget
async def scenario():
async with endpoint(transport, monkeypatch, request_limit=limit, chunk_target=frame_bytes) as ctx:
await send(ctx, images, audio=audio)
items = queued_items(ctx.queue)
assert len(ctx.requests) == 3 + int(separate)
assert [item["is_last"] for item in items] == [False] * (len(items) - 1) + [True]
assert all(item["audio"] is None for item in items[:-1])
assert items[-1]["audio"]["sample_rate"] == audio["sample_rate"]
assert_exact(items[-1]["audio"]["waveform"], audio["waveform"])
assert_exact(torch.cat([item["tensor"] for item in items if item["tensor"] is not None]), images)
assert all(record["bytes"] <= limit for record in ctx.requests)
if separate:
assert items[-1]["tensor"] is None
assert ctx.requests[-1]["content_type"] == "application/json"
asyncio.run(scenario())
+112
View File
@@ -0,0 +1,112 @@
"""Real Collector aggregation of typed bridge deliveries and direct results.
Only Comfy's host services are stubbed by the shared Collector loader. Tensor
normalization, participant ordering, queue consumption and cleanup are real.
"""
import asyncio
import json
import types
from unittest.mock import AsyncMock
import pytest
import torch
from test_collector_list_inputs import _load_collector_module
@pytest.fixture
def collector_module():
module = _load_collector_module()
module.prompt_server.distributed_jobs_lock = asyncio.Lock()
module.prompt_server.distributed_serverless = types.SimpleNamespace(cancel=AsyncMock())
return module
def native_media(value=0):
image = torch.arange(24, dtype=torch.float64).reshape(1, 2, 3, 4) / 997 + value
image[0, 0, 0, 0] = -0.0
audio = {"waveform": torch.arange(6, dtype=torch.float64).reshape(1, 2, 3) / 997 + value,
"sample_rate": 24000}
return {"images": image, "audio": audio}
def assert_exact(actual, expected):
assert actual.dtype == expected.dtype and actual.shape == expected.shape
assert torch.equal(actual.contiguous().view(torch.uint8), expected.contiguous().view(torch.uint8))
def collect_queue(module, messages, participants, *, local=None, delegate=True):
queue = asyncio.Queue()
for message in messages:
queue.put_nowait(message)
module.prompt_server.distributed_pending_jobs["typed-job"] = queue
values = local or {"images": None, "audio": None}
return asyncio.run(asyncio.wait_for(module.DistributedCollectorNode().execute(
**values, multi_job_id="typed-job", enabled_worker_ids=json.dumps(participants),
delegate_only=delegate), timeout=2))
@pytest.mark.parametrize("delegate", [False, True])
@pytest.mark.parametrize("socket_set", [("images",), ("audio",), ("images", "audio")])
def test_typed_and_direct_results_preserve_batch_and_audio_order(collector_module, delegate, socket_set):
module = collector_module
local, typed_a, typed_b, direct = [native_media(n) for n in range(4)]
typed = {socket: [typed_a[socket], typed_b[socket]] for socket in socket_set}
direct_message = {"worker_id": "direct", "is_last": True}
if "images" in socket_set:
direct_message.update(tensor=direct["images"], image_index=0)
if "audio" in socket_set:
direct_message["audio"] = direct["audio"]
messages = [direct_message, {"worker_id": "serverless:pool:1", "media": typed, "is_last": True}]
# Delivery order differs from participant order. Duplicate configured IDs
# must not duplicate a participant's data or leave collection waiting.
participants = ["serverless:pool:1", "direct", "serverless:pool:1"]
local_inputs = {socket: local[socket] if socket in socket_set else None for socket in ("images", "audio")}
images, audio = collect_queue(module, messages, participants, local=local_inputs, delegate=delegate)
expected = ([] if delegate else [local]) + [typed_a, typed_b, direct]
if "images" in socket_set:
assert_exact(images, torch.cat([row["images"] for row in expected]))
else:
assert images is None
if "audio" in socket_set:
assert audio["sample_rate"] == 24000
assert_exact(audio["waveform"], torch.cat([row["audio"]["waveform"] for row in expected], dim=-1))
else:
assert torch.equal(audio["waveform"], module.DistributedCollectorNode.EMPTY_AUDIO["waveform"])
assert module.prompt_server.distributed_pending_jobs == {}
module.prompt_server.distributed_serverless.cancel.assert_not_awaited()
@pytest.mark.parametrize("media", [{}, {"texts": ["not a Collector socket"]}, {"images": []},
{"audio": [None]}, {"images": "not a list"},
{"images": [torch.zeros(2, 2, 3)]}])
def test_invalid_typed_delivery_cancels_and_removes_queue(collector_module, media):
module = collector_module
with pytest.raises(RuntimeError, match="media|items|IMAGE"):
collect_queue(module, [{"worker_id": "serverless:pool:1", "media": media, "is_last": True}],
["serverless:pool:1"])
assert module.prompt_server.distributed_pending_jobs == {}
module.prompt_server.distributed_serverless.cancel.assert_awaited_once_with(collector_job_id="typed-job")
@pytest.mark.parametrize("failure", ["nonfinal", "unexpected", "duplicate", "provider-error", "shape"])
def test_typed_delivery_protocol_failures_are_not_partial_success(collector_module, failure):
media = {"images": [native_media()["images"]]}
first = {"worker_id": "serverless:pool:1", "media": media, "is_last": True}
messages = [first]
participants = ["serverless:pool:1", "serverless:pool:2"]
if failure == "nonfinal":
first["is_last"] = False
elif failure == "unexpected":
first["worker_id"] = "other"
elif failure == "duplicate":
messages.append(dict(first))
elif failure == "provider-error":
messages.append({"worker_id": "serverless:pool:2", "serverless_error": "offline failure", "is_last": True})
else:
messages.append({"worker_id": "serverless:pool:2", "media": {"images": [torch.zeros(1, 4, 5, 3)]},
"is_last": True})
with pytest.raises(RuntimeError, match="atomic|Unexpected|Duplicate|failed|shape mismatch"):
collect_queue(collector_module, messages, participants)
assert collector_module.prompt_server.distributed_pending_jobs == {}
collector_module.prompt_server.distributed_serverless.cancel.assert_awaited_once_with(collector_job_id="typed-job")
+19
View File
@@ -472,6 +472,25 @@ class DispatchSelectionTests(unittest.IsolatedAsyncioTestCase):
with self.assertRaisesRegex(RuntimeError, "execution_targets.*upgrade"):
await dispatch.preflight_image_list_workers([{"id": "worker"}])
async def test_image_list_value_results_requires_exact_true_only_for_overrides(self):
for value in (None, False, 1, "true", [], True):
capability = _valid_capability()
if value is not None:
capability['value_results'] = value
session = _FakeSession({('worker', 'capabilities'): (200, capability),
('worker', 'queue'): (200, _valid_queue())})
async def fake_session():
return session
with patch.object(dispatch, 'get_client_session', side_effect=fake_session), patch.object(
dispatch, 'build_worker_url', side_effect=lambda worker, path: f"http://{worker['id']}{path}"
):
self.assertEqual(await dispatch.preflight_image_list_workers([{'id': 'worker'}]), {'worker': True})
if value is True:
self.assertEqual(await dispatch.preflight_image_list_workers(
[{'id': 'worker'}], require_value_results=True), {'worker': True})
else:
with self.assertRaisesRegex(RuntimeError, 'value_results.*upgrade'):
await dispatch.preflight_image_list_workers([{'id': 'worker'}], require_value_results=True)
async def test_image_list_preflight_fails_closed_for_malformed_capabilities(self):
malformed = {"supported": True, "protocol_version": 2, "node_ids": None}
session = _FakeSession({("worker", "capabilities"): (200, malformed)})
@@ -0,0 +1,20 @@
"""Run the real framework's CPU-only private-cache retention regression."""
import os
from pathlib import Path
import subprocess
import sys
import pytest
def test_native_local_cache_retention():
core = os.environ.get('COMFYUI_SOURCE_ROOT')
if not core:
pytest.skip('Set COMFYUI_SOURCE_ROOT for CPU-only real-framework validation')
helper = Path(__file__).parent / 'helpers' / 'local_cache_retention_check.py'
env = dict(os.environ, CUDA_VISIBLE_DEVICES='', HIP_VISIBLE_DEVICES='')
env.pop('PYTHONPATH', None)
result = subprocess.run([sys.executable, str(helper), core], env=env,
text=True, capture_output=True, timeout=90)
assert result.returncode == 0, result.stdout + result.stderr
assert 'PASS: cross-run native cache reuse' in result.stdout
+113
View File
@@ -191,6 +191,50 @@ class MasterRuntimeTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(result[2], [str(i) for i in range(9)])
self.preflight.assert_awaited_once_with(self.workers, require_idle=False)
async def test_busy_direct_peer_does_not_block_selected_cloud_slots(self):
self.workers.append({'id': 'worker', 'type': 'remote'})
context = self.create(participates=False, count=3)
self.config['workers'] = self.workers + [{'id': 'pool', 'type': 'serverless'}]
self.state.worker_ids.append('pool')
self.state.original_prompt['end']['inputs']['value_10'] = ['start', 2]
fixture = self
cloud_calls = []
async def busy_preflight(workers, **kwargs):
if kwargs['require_idle']:
raise RuntimeError('No selected feature-compatible direct worker is idle')
return {}
self.preflight.side_effect = busy_preflight
class Controller:
slots = [('serverless', 'pool', 0)]
close = AsyncMock()
async def dispatch(self, slot, index):
cloud_calls.append(index)
return {'index': index}
def poll(self, inflight):
slot = self.slots[0]
if slot in inflight:
index = inflight[slot]['index']
return slot, index, {'image': fixture.items[index]['image'],
'values': {'value_10': index}}
return None
controller = Controller()
create_slots = AsyncMock(return_value=controller)
with patch.dict(sys.modules, {'_master_slot_tests.image_list.serverless':
types.SimpleNamespace(create_serverless_slots=create_slots)}):
result = await self.rt.run_image_list_job(context, include_values=True)
self.assertEqual(cloud_calls, [0, 1, 2])
self.assertEqual(result[3], [{'value_1': str(i), 'value_10': i} for i in range(3)])
self.preflight.assert_awaited_once_with(self.workers, require_idle=False, require_value_results=True)
self.network.get_client_session.assert_not_awaited()
controller.close.assert_awaited_once()
self.assertFalse(self.local_calls or self.dispatched or self.rt._MASTER_JOBS)
async def test_orchestrator_mode_never_instantiates_local_slot(self):
self.workers.append({'id': 'worker', 'type': 'remote'})
await self.rt.run_image_list_job(self.create(participates=False))
@@ -199,6 +243,73 @@ class MasterRuntimeTests(unittest.IsolatedAsyncioTestCase):
with self.assertRaisesRegex(RuntimeError, 'worker|fallback'):
self.rt.create_master_job('start', 'end', graph(), self.items, [], '')
async def test_values_reconstructed_in_source_order_without_transporting_originals(self):
self.workers.append({'id': 'remote', 'type': 'remote'})
context = self.create(count=6)
tensor = torch.ones(4, 2, 2, 3, dtype=torch.float64)
for index, item in enumerate(self.items):
item['values'].update(value_2=tensor, value_3=True, value_4=index / 8)
self.local_delay = 0.1 # Remote results arrive ahead of master item zero.
with patch.object(self.rt, 'encode_result', side_effect=AssertionError('originals must not return')):
result = await self.rt.run_image_list_job(context, include_values=True)
self.assertFalse(self.rt._MASTER_JOBS)
self.assertEqual(len(result), 4)
for index, row in enumerate(result[3]):
self.assertEqual(row['value_1'], str(index))
self.assertIs(row['value_2'], tensor)
self.assertIs(row['value_3'], True)
self.assertEqual(row['value_4'], index / 8)
self.assertTrue(self.dispatched and self.local_calls)
async def test_overrides_merge_add_and_recheck_capability_before_every_item(self):
self.workers.append({'id': 'remote', 'type': 'remote'})
context = self.create(participates=False, count=3)
self.state.original_prompt['end']['inputs'].update(value_1=['start', 4], value_10=['start', 2])
original_set = self.rt.set_master_result
def override(context, index, token, result):
result['values'] = {'value_1': f'new-{index}', 'value_10': index}
return original_set(context, index, token, result)
with patch.object(self.rt, 'set_master_result', side_effect=override):
result = await self.rt.run_image_list_job(context, include_values=True)
self.assertEqual(result[3], [{'value_1': f'new-{index}', 'value_10': index} for index in range(3)])
self.assertEqual(self.preflight.await_count, 4)
self.assertTrue(all(call.kwargs['require_value_results'] is True
for call in self.preflight.await_args_list))
self.assertEqual(self.items[0]['values'], {'value_1': '0'})
async def test_override_capability_downgrade_stops_before_upload(self):
self.workers.append({'id': 'remote', 'type': 'remote'})
context = self.create(participates=False)
self.state.original_prompt['end']['inputs']['value_1'] = ['start', 4]
self.preflight.side_effect = [{'remote': True}, RuntimeError('value_results unsupported')]
with patch.object(self, 'post', wraps=self.post) as upload:
with self.assertRaisesRegex(RuntimeError, 'value_results'):
await self.rt.run_image_list_job(context, include_values=True)
upload.assert_not_called()
self.assertFalse(self.dispatched or self.rt._MASTER_JOBS)
async def test_missing_expected_override_fails_without_partial_output(self):
self.workers.append({'id': 'remote', 'type': 'remote'})
context = self.create(participates=False)
self.state.original_prompt['end']['inputs']['value_1'] = ['start', 4]
with self.assertRaisesRegex(ValueError, 'missing'):
await self.rt.run_image_list_job(context, include_values=True)
self.assertFalse(self.rt._MASTER_JOBS)
async def test_values_failure_and_cancellation_return_no_partial_lists(self):
self.local_error = RuntimeError('branch failed')
with self.assertRaisesRegex(RuntimeError, 'branch failed'):
await self.rt.run_image_list_job(self.create(), include_values=True)
self.local_error = None
self.local_started.clear()
self.local_delay = 0.1
task = asyncio.create_task(self.rt.run_image_list_job(self.create(), include_values=True))
await self.local_started.wait()
task.cancel()
with self.assertRaises(self.rt.ImageListCancelledError):
await task
self.assertFalse(self.rt._MASTER_JOBS)
async def test_local_failure_surfaces_and_reclaims_staged_token(self):
self.local_error = RuntimeError('local branch failed')
with self.assertRaisesRegex(RuntimeError, 'local branch failed'):
@@ -373,6 +484,8 @@ def real_framework_probe(comfy_root):
rt.remove_master_job(context)
for behavior in ('normal', 'expand', 'error', 'cancel', 'interrupt', 'sync', 'already-interrupted'):
# Independent lifecycle cases start cold; retention is tested separately.
local._RETAINED_OUTPUTS = None
mode[0] = behavior
prompt['op']['class_type'] = 'Expanded' if behavior == 'expand' else 'Operation'
loads.clear()
+160 -5
View File
@@ -1,8 +1,9 @@
"""Image-list participation uses the existing master orchestrator setting."""
import json
import sys
import types
import unittest
from unittest.mock import AsyncMock, patch
from unittest.mock import AsyncMock, Mock, patch
from test_serverless_integration import PKG, load_modules
@@ -62,11 +63,21 @@ class MasterSelectionTests(unittest.IsolatedAsyncioTestCase):
self.q.dispatch_worker_prompt.assert_not_awaited()
self.assertEqual(self.ps.distributed_pending_jobs, {})
async def test_end_overrides_require_value_capability_before_master_queue(self):
self.prompt['3']['inputs']['value_10'] = ['2', 2]
await self.queue()
self.assertIs(self.preflight.call_args.kwargs['require_value_results'], True)
self.q.queue_prompt_payload.reset_mock()
self.preflight.side_effect = RuntimeError('value_results unsupported')
with self.assertRaisesRegex(RuntimeError, 'value_results'):
await self.queue()
self.q.queue_prompt_payload.assert_not_awaited()
async def test_orchestrator_setting_excludes_master_processing(self):
self.config["settings"]["master_delegate_only"] = True
await self.queue()
self.assertIs(self.prepared_inputs()["image_list_master_participates"], False)
self.assertTrue(self.preflight.call_args.kwargs["require_idle"])
self.assertIs(self.preflight.call_args.kwargs["require_idle"], False)
# Upstream source stays on master even in orchestrator-only mode.
self.assertIn("1", self.q.queue_prompt_payload.call_args.args[0])
@@ -97,10 +108,154 @@ class MasterSelectionTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), [])
self.assertIs(self.prepared_inputs()["image_list_master_participates"], True)
async def test_busy_workers_do_not_prevent_enabled_master(self):
self.preflight.return_value = {}
def use_real_preflight(self, queues):
"""Exercise capability/queue parsing with read-only fake HTTP responses."""
from test_dispatch_selection import dispatch, _FakeSession, _valid_capability
responses = {(worker_id, "capabilities"): (200, _valid_capability())
for worker_id in queues}
responses.update({(worker_id, "queue"): (200, queue)
for worker_id, queue in queues.items()})
session = _FakeSession(responses)
for target, value in (
("get_client_session", AsyncMock(return_value=session)),
("build_worker_url", lambda worker, path: f"http://{worker['id']}{path}"),
):
patched = patch.object(dispatch, target, value)
patched.start()
self.addCleanup(patched.stop)
self.preflight = AsyncMock(wraps=dispatch.preflight_image_list_workers)
sys.modules[PKG + ".api.orchestration.dispatch"].preflight_image_list_workers = self.preflight
return responses
async def run_prepared_selection(self):
"""Run real runtime selection up to the scheduler, without any item I/O."""
rt = sys.modules[PKG + ".image_list.runtime"]
inputs = self.prepared_inputs()
image = object()
context = rt.create_master_job(
"2", "3", self.q.queue_prompt_payload.call_args.args[0],
[{"image": image, "index": 0}],
json.loads(inputs["image_list_enabled_worker_ids"]),
inputs["image_list_master_callback"],
master_participates=inputs["image_list_master_participates"],
explicit_targets=True,
)
self.addCleanup(rt.remove_master_job, context)
self.scheduler = AsyncMock(return_value=[{"image": image}])
self.local_slot = Mock(return_value=types.SimpleNamespace(close=AsyncMock()))
self.runtime_session = AsyncMock(return_value=object())
config = sys.modules[PKG + ".utils.config"]
network = sys.modules[PKG + ".utils.network"]
with patch.object(config, "load_config", lambda: self.config), \
patch.object(config, "get_worker_timeout_seconds", lambda: 60, create=True), \
patch.object(config, "get_image_list_job_timeout_seconds", lambda: 0, create=True), \
patch.object(network, "get_client_session", self.runtime_session, create=True), \
patch.object(network, "build_worker_url", Mock(), create=True), \
patch.object(rt, "dynamic_schedule", self.scheduler), \
patch.dict(sys.modules, {PKG + ".image_list.local_executor": types.SimpleNamespace(
LocalExecutionSlot=self.local_slot)}):
await rt.run_image_list_job(context)
self.assertIsNone(rt.get_master_job(context))
return self.scheduler.call_args.args[1]
async def test_busy_worker_at_enqueue_is_available_to_scheduler_when_idle_at_start(self):
responses = self.use_real_preflight({
"worker": {"queue_running": [[1, "prompt-a"]], "queue_pending": []},
})
result = await self.queue()
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), ["worker"])
self.assertEqual(result[2], 1) # Candidates, not a promise of execution-time availability.
self.q.dispatch_worker_prompt.assert_not_awaited()
responses[("worker", "queue")] = (200, {"queue_running": [], "queue_pending": []})
# A newly enabled worker is not implicitly added to this queued selection.
self.config["workers"].append({"id": "later", "enabled": True})
slots = await self.run_prepared_selection()
self.assertEqual([slot for slot in slots if isinstance(slot, str)], ["worker"])
self.assertEqual(len(slots), 2)
self.assertEqual(self.preflight.await_count, 2)
self.assertEqual(self.preflight.await_args_list[0].args, self.preflight.await_args_list[1].args)
async def test_worker_still_busy_at_start_is_excluded_from_scheduler(self):
self.use_real_preflight({
"worker": {"queue_running": [], "queue_pending": [[1, "prompt-a"]]},
})
await self.queue()
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), [])
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), ["worker"])
slots = await self.run_prepared_selection()
self.assertEqual(len(slots), 1)
self.assertNotIn("worker", slots)
self.assertEqual(self.preflight.await_count, 2)
self.runtime_session.assert_not_awaited()
async def test_explicit_empty_selection_stays_master_only_at_start(self):
self.use_real_preflight({"worker": {"queue_running": [], "queue_pending": []}})
await self.queue(enabled_worker_ids=[])
slots = await self.run_prepared_selection()
self.assertEqual(len(slots), 1)
self.assertNotIn("worker", slots)
self.preflight.assert_not_awaited()
self.runtime_session.assert_not_awaited()
async def test_busy_incompatible_worker_fails_before_master_queue(self):
responses = self.use_real_preflight({
"worker": {"queue_running": [[1, "prompt-a"]], "queue_pending": []},
})
capability = responses[("worker", "capabilities")][1]
for field, invalid, message in (
("protocol_version", 1, "incompatible"),
("execution_targets", False, "execution_targets"),
("value_results", False, "value_results"),
):
with self.subTest(field=field), patch.dict(capability, {field: invalid}):
if field == "value_results":
self.prompt["3"]["inputs"]["value_10"] = ["2", 2]
with self.assertRaisesRegex(RuntimeError, message):
await self.queue()
self.q.queue_prompt_payload.assert_not_awaited()
self.q.dispatch_worker_prompt.assert_not_awaited()
async def test_orchestrator_with_all_workers_busy_queues_for_idle_start(self):
responses = self.use_real_preflight({
"worker": {"queue_running": [[1, "prompt-a"]], "queue_pending": []},
})
result = await self.queue(delegate_master=True)
self.q.queue_prompt_payload.assert_awaited_once()
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), ["worker"])
self.assertIs(self.prepared_inputs()["image_list_master_participates"], False)
self.assertEqual(result[2], 1)
self.q.dispatch_worker_prompt.assert_not_awaited()
responses[("worker", "queue")] = (200, {"queue_running": [], "queue_pending": []})
self.assertEqual(await self.run_prepared_selection(), ["worker"])
self.local_slot.assert_not_called()
self.assertEqual(self.preflight.await_count, 2)
self.assertIs(self.preflight.await_args_list[0].kwargs["require_idle"], False)
self.assertIs(self.preflight.await_args_list[1].kwargs["require_idle"], True)
async def test_orchestrator_rechecks_availability_without_master_fallback(self):
responses = self.use_real_preflight({"worker": {"queue_running": [], "queue_pending": []}})
await self.queue(delegate_master=True)
responses[("worker", "queue")] = (200, {"queue_running": [[1, "other"]], "queue_pending": []})
with self.assertRaisesRegex(RuntimeError, "No selected feature-compatible direct worker is idle"):
await self.run_prepared_selection()
self.assertEqual(self.preflight.await_count, 2)
self.scheduler.assert_not_awaited()
self.local_slot.assert_not_called()
async def test_orchestrator_retains_busy_candidates_when_another_worker_is_idle(self):
self.config["workers"].append({"id": "idle", "enabled": True})
responses = self.use_real_preflight({
"worker": {"queue_running": [[1, "prompt-a"]], "queue_pending": []},
"idle": {"queue_running": [], "queue_pending": []},
})
await self.queue(delegate_master=True)
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), ["worker", "idle"])
responses[("worker", "queue")] = (200, {"queue_running": [], "queue_pending": []})
responses[("idle", "queue")] = (200, {"queue_running": [[1, "other"]], "queue_pending": []})
self.assertEqual(await self.run_prepared_selection(), ["worker"])
self.local_slot.assert_not_called()
async def test_orchestrator_without_workers_fails_before_queue(self):
self.config["settings"]["master_delegate_only"] = True
@@ -0,0 +1,536 @@
"""Stable content signatures with fresh, expiring delivery capabilities."""
import asyncio
import copy
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import types
from unittest.mock import AsyncMock
import pytest
import torch
from image_list import runtime as rt
ROOT = Path(__file__).resolve().parents[1]
def item():
return rt.normalize_image_list_inputs(
torch.arange(18, dtype=torch.float64).reshape(1, 2, 3, 3),
torch.zeros(1, 2, 3, dtype=torch.bfloat16),
{'value_1': 'original', 'value_2': True, 'value_3': 7, 'value_4': 0.25,
'value_10': torch.ones(2, 3, dtype=torch.float16)})[0]
def graph():
return {
'source': {'class_type': 'Source', 'inputs': {}},
'start': {'class_type': rt.START_CLASS, 'inputs': {
'images': ['source', 0], 'image_list_execution_id': 'transient',
'image_list_workflow_meta': {'job': 'metadata'},
'image_list_enabled_worker_ids': '["worker"]',
'image_list_master_callback': 'http://old-master',
'values.value_1': 'inherited', 'worker_context_id': 'old-context'}},
'op': {'class_type': 'Operation', 'inputs': {'image': ['start', 0]}},
'end': {'class_type': rt.END_CLASS, 'inputs': {
'images': ['op', 0], 'context': ['start', 3], 'value_3': ['start', 2]}},
}
def worker_prompt(token, context, data, *, local=True):
prompt = graph()
result = rt.build_worker_prompt(prompt, rt.validate_image_list_regions(prompt)[0],
token, context, data['index'], 'http://master',
item_identity=rt.item_content_identity(data))
if local:
result['start']['inputs']['image_list_local_execution'] = True
result['end']['inputs']['image_list_local_execution'] = True
return result
@pytest.fixture(autouse=True)
def clean_state():
yield
rt._WORKER_ITEMS.clear()
rt._MASTER_JOBS.clear()
@pytest.fixture
def boundaries(monkeypatch):
"""Load only the boundary module; restore every replaced module on exit."""
name = '_native_cache_identity_tests'
for suffix in ('', '.nodes', '.utils', '.image_list', '.api'):
module = types.ModuleType(name + suffix)
module.__path__ = [str(ROOT / suffix.lstrip('.').replace('.', '/'))]
monkeypatch.setitem(sys.modules, name + suffix, module)
monkeypatch.setitem(sys.modules, name + '.image_list.runtime', rt)
io = types.SimpleNamespace(ComfyNode=type('ComfyNode', (), {}),
NodeOutput=lambda *args: args)
for cls in ('Image', 'Mask', 'String', 'Int', 'Float', 'Boolean'):
setattr(io, cls, object())
monkeypatch.setitem(sys.modules, 'comfy_api', types.ModuleType('comfy_api'))
monkeypatch.setitem(sys.modules, 'comfy_api.v0_0_2', types.SimpleNamespace(io=io))
monkeypatch.setitem(sys.modules, name + '.utils.async_helpers',
types.SimpleNamespace(run_async_in_server_loop=lambda coro, **kw: asyncio.run(coro)))
spec = importlib.util.spec_from_file_location(name + '.nodes.image_list', ROOT / 'nodes/image_list.py')
module = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, spec.name, module)
spec.loader.exec_module(module)
return module
def start(boundaries, prompt):
boundaries.DistributedImageListStart.hidden = types.SimpleNamespace(prompt=prompt, unique_id='start')
return boundaries.DistributedImageListStart.execute(**prompt['start']['inputs'])
def end(boundaries, prompt, cached):
boundaries.DistributedImageListEnd.hidden = types.SimpleNamespace(prompt=prompt, unique_id='end')
inputs = {k: v for k, v in prompt['end']['inputs'].items()
if k not in ('images', 'context', 'value_3')}
return boundaries.DistributedImageListEnd.execute(
images=cached[0], context=cached[3], value_3=cached[2], **inputs)
def stage(data):
token = rt.create_worker_task_token()
rt.store_worker_item(token, rt.encode_item(data))
return token
def master(data, token):
context = rt.create_master_job('start', 'end', graph(), [data], ['worker'], '')
rt.register_assignment(context, data['index'], 'worker', token)
return context
def test_identity_is_exact_typed_order_independent_and_not_transport_bounded(monkeypatch):
original = item()
identity = rt.item_content_identity(original)
assert rt.is_valid_item_identity(identity)
reordered = copy.deepcopy(original)
reordered['values'] = dict(reversed(list(reordered['values'].items())))
assert rt.item_content_identity(reordered) == identity
strided = copy.deepcopy(original)
strided['image'] = original['image'].transpose(1, 2).contiguous().transpose(1, 2)
assert not strided['image'].is_contiguous()
assert rt.item_content_identity(strided) == identity
monkeypatch.setattr(rt, 'MAX_PAYLOAD_BYTES', 1)
with pytest.raises(ValueError, match='transport limit'):
rt.encode_item(original)
assert rt.item_content_identity(original) == identity
scalar_identities = set()
for value in (True, 1, 1.0, '1', 0.0, -0.0):
changed = copy.deepcopy(original)
changed['values']['value_2'] = value
scalar_identities.add(rt.item_content_identity(changed))
assert len(scalar_identities) == 6
@pytest.mark.parametrize('change', ['image', 'mask', 'value', 'index', 'dtype', 'shape', 'slot', 'missing-mask'])
def test_every_item_component_invalidates_identity(change):
original, changed = item(), item()
if change in ('image', 'mask'):
changed[change].reshape(-1)[0] = 1
elif change == 'value':
changed['values']['value_10'][0, 0] = 2
elif change == 'index':
changed['index'] = 1
elif change == 'dtype':
changed['image'] = changed['image'].float()
elif change == 'shape':
changed['values']['value_10'] = changed['values']['value_10'].reshape(3, 2)
elif change == 'slot':
changed['values']['value_9'] = changed['values'].pop('value_10')
else:
changed['mask'] = None
assert rt.item_content_identity(original) != rt.item_content_identity(changed)
def test_float64_low_bits_and_bfloat16_bytes_are_not_quantized():
data, changed = item(), item()
changed['image'].reshape(-1)[0] = torch.nextafter(torch.tensor(0., dtype=torch.float64),
torch.tensor(1., dtype=torch.float64))
assert rt.item_content_identity(data) != rt.item_content_identity(changed)
changed = item()
changed['mask'].view(torch.int16).reshape(-1)[0] = 1
assert rt.item_content_identity(data) != rt.item_content_identity(changed)
def test_stable_start_replaced_wholesale_and_legacy_graph_preserved():
prompt = graph()
frozen = copy.deepcopy(prompt)
region = rt.validate_image_list_regions(prompt)[0]
legacy = rt.build_worker_prompt(prompt, region, 'a', 'ctx-a', 0, 'http://a')
assert legacy['start']['inputs']['worker_item_token'] == 'a'
assert legacy['start']['inputs']['image_list_execution_id'] == 'transient'
identity = rt.item_content_identity(item())
a = rt.build_worker_prompt(prompt, region, 'a', 'ctx-a', 0, 'http://a', item_identity=identity)
b = rt.build_worker_prompt(prompt, region, 'b', 'ctx-b', 0, 'http://b', item_identity=identity)
assert a['start']['inputs'] == b['start']['inputs'] == {'worker_item_identity': identity}
assert a['end']['inputs']['worker_context_id'] == 'ctx-a'
assert b['end']['inputs']['worker_result_token'] == 'b'
assert b['end']['inputs']['worker_item_identity'] == identity
assert b['end']['inputs']['value_3'] == ['start', 2]
assert prompt == frozen
@pytest.mark.parametrize('bad', ['', 'sha256:' + 'z' * 64, 'a' * 64, False, ['bad']])
def test_malformed_identity_rejected(bad):
assert not rt.is_valid_item_identity(bad)
prompt = graph()
with pytest.raises((ValueError, TypeError), match='identity'):
rt.build_worker_prompt(prompt, rt.validate_image_list_regions(prompt)[0],
'token', 'context', 0, '', item_identity=bad)
def test_cache_miss_then_hit_delivers_current_context_token_and_values(boundaries):
data = item()
a = stage(data)
context_a = master(data, a)
prompt_a = worker_prompt(a, context_a, data)
cached = start(boundaries, prompt_a)
assert cached[3] == [rt.item_content_identity(data)]
assert 'image' not in rt._WORKER_ITEMS[a][1] # only a small, expiring receipt remains
end(boundaries, prompt_a, cached)
assert a not in rt._WORKER_ITEMS
assert rt.get_master_job(context_a).results[0]['values'] == {'value_3': 0}
b = stage(data)
context_b = master(data, b)
prompt_b = worker_prompt(b, context_b, data)
assert prompt_a['start'] == prompt_b['start']
# A native hit does NOT execute Start; End must consume this current upload.
end(boundaries, prompt_b, cached)
assert b not in rt._WORKER_ITEMS
state = rt.get_master_job(context_b)
assert state.expected_tokens[0] == b
assert torch.equal(state.results[0]['image'], data['image'])
assert rt._collect_output_lists([state.results[0]], state=state, include_values=True)[3][0]['value_3'] == 0
with pytest.raises(RuntimeError, match='missing|expired|consumed'):
end(boundaries, prompt_b, cached)
@pytest.mark.parametrize('miss', [False, True])
def test_expired_item_cannot_deliver_even_with_cached_start(boundaries, miss):
data = item()
token = stage(data)
context = master(data, token)
prompt = worker_prompt(token, context, data)
cached = start(boundaries, prompt) if miss else ([data['image']], [data['mask']], [0], [rt.item_content_identity(data)])
_, staged = rt._WORKER_ITEMS[token]
rt._WORKER_ITEMS[token] = (0, staged)
with pytest.raises(RuntimeError, match='missing|expired|consumed'):
end(boundaries, prompt, cached)
assert not rt.get_master_job(context).received_indexes
assert token not in rt._WORKER_ITEMS
def test_identity_mismatch_on_start_and_on_cache_hit_fails_closed(boundaries):
data, different = item(), item()
different['image'][0, 0, 0, 0] = 3
for hit in (False, True):
token = stage(different)
context = master(different, token)
prompt = worker_prompt(token, context, data)
cached = ([data['image']], [data['mask']], [0], [rt.item_content_identity(data)])
with pytest.raises((ValueError, RuntimeError), match='identity'):
end(boundaries, prompt, cached) if hit else start(boundaries, prompt)
assert not rt.get_master_job(context).received_indexes
rt.delete_worker_item(token)
@pytest.mark.parametrize('change', ['ambiguous', 'wrong-link', 'wrong-start-id', 'end-identity', 'end-index'])
def test_start_requires_exact_paired_end_binding(boundaries, change):
data = item()
token = stage(data)
prompt = worker_prompt(token, 'context', data)
if change == 'ambiguous':
prompt['second-end'] = copy.deepcopy(prompt['end'])
elif change == 'wrong-link':
prompt['end']['inputs']['context'] = ['start', 2]
elif change == 'wrong-start-id':
prompt['renamed'] = prompt.pop('start')
elif change == 'end-identity':
prompt['end']['inputs']['worker_item_identity'] = 'sha256:' + '0' * 64
else:
prompt['end']['inputs']['worker_item_index'] = 1
boundaries.DistributedImageListStart.hidden = types.SimpleNamespace(prompt=prompt, unique_id='start')
with pytest.raises((RuntimeError, ValueError), match='paired|identity|index'):
boundaries.DistributedImageListStart.execute(worker_item_identity=rt.item_content_identity(data),
image_list_local_execution=True)
def test_end_checks_stable_context_before_delivering(boundaries):
data = item()
token = stage(data)
context = master(data, token)
prompt = worker_prompt(token, context, data)
cached = list(start(boundaries, prompt))
cached[3] = ['sha256:' + '0' * 64]
with pytest.raises((RuntimeError, ValueError), match='identity|context'):
end(boundaries, prompt, cached)
assert not rt.get_master_job(context).received_indexes
def test_legacy_worker_path_still_consumes_and_delivers_job_context(boundaries):
data = item()
token = stage(data)
context = master(data, token)
prompt = graph()
prompt = rt.build_worker_prompt(prompt, rt.validate_image_list_regions(prompt)[0], token, context, 0, '')
prompt['start']['inputs']['image_list_local_execution'] = True
prompt['end']['inputs']['image_list_local_execution'] = True
cached = start(boundaries, prompt)
assert cached[3] == [context]
assert token not in rt._WORKER_ITEMS
end(boundaries, prompt, cached)
assert rt.get_master_job(context).received_indexes == {0}
@pytest.mark.parametrize('fmt', ['exact', 'png'])
def test_remote_cache_hit_submits_fresh_callback_metadata(boundaries, monkeypatch, fmt):
data = item()
token = stage(data)
prompt = worker_prompt(token, 'fresh-context', data, local=False)
prompt['end']['inputs'].update(transfer_format=fmt, transfer_quality=73)
submit = AsyncMock()
monkeypatch.setattr(boundaries, 'submit_worker_result', submit)
cached = ([data['image']], [data['mask']], [0], [rt.item_content_identity(data)])
result = end(boundaries, prompt, cached)
assert len(result) == 4
assert submit.await_args.args[:4] == ('http://master', token, 'fresh-context', 0)
assert submit.await_args.args[4]['values']['value_3'] == {'type': 'INT', 'value': 0}
payload = submit.await_args.args[4]
if fmt == 'png':
assert payload['image']['descriptor']['format'] == fmt
assert payload['image']['descriptor']['quality'] == 73
else:
assert 'encoding' not in payload['image']
assert token not in rt._WORKER_ITEMS
def test_item_route_returns_server_computed_identity(boundaries, monkeypatch):
from aiohttp import web
name = '_native_cache_identity_tests'
server = types.SimpleNamespace(PromptServer=types.SimpleNamespace(instance=types.SimpleNamespace(routes=web.RouteTableDef())))
monkeypatch.setitem(sys.modules, 'server', server)
spec = importlib.util.spec_from_file_location(name + '.api.image_list_routes', ROOT / 'api/image_list_routes.py')
module = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, spec.name, module)
spec.loader.exec_module(module)
data = item()
token = rt.create_worker_task_token()
monkeypatch.setattr(module, '_json_body', AsyncMock(return_value={
'token': token, 'context_id': 'context', 'item': rt.encode_item(data),
'item_identity': 'sha256:' + '0' * 64}))
response = asyncio.run(module.image_list_item_endpoint(object()))
assert response.status == 200
assert json.loads(response.body)['item_identity'] == rt.item_content_identity(data)
assert len(rt._WORKER_ITEMS[token]) == 2
@pytest.mark.parametrize('responses', [
('valid', 'valid'), ('valid', 'absent'), ('invalid', 'valid'),
('bad-json', 'valid'), ('non-object', 'valid'),
])
def test_runtime_opts_in_per_upload_and_can_downgrade(monkeypatch, responses):
name = '_native_cache_identity_runtime'
for suffix in ('', '.image_list', '.api', '.api.orchestration', '.utils'):
module = types.ModuleType(name + suffix)
module.__path__ = [str(ROOT / suffix.lstrip('.').replace('.', '/'))]
monkeypatch.setitem(sys.modules, name + suffix, module)
spec = importlib.util.spec_from_file_location(name + '.image_list.runtime', ROOT / 'image_list/runtime.py')
runtime = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, spec.name, runtime)
spec.loader.exec_module(runtime)
workers = [{'id': 'worker', 'type': 'local'}]
dispatched, uploads = [], []
preflight = AsyncMock(return_value={'worker': True})
sync_media = AsyncMock()
class Response:
status = 200
def __init__(self, data):
self.data = data
async def __aenter__(self):
return self
async def __aexit__(self, *args):
pass
async def json(self):
if self.data == 'bad-json':
raise ValueError('old response is not JSON')
return self.data
def raise_for_status(self):
pass
class Session:
def post(self, url, *, data, **kw):
assert url.endswith('/distributed/image-list/item')
body = json.loads(data)
identity = runtime.store_worker_item(body['token'], body['item'])
mode = responses[len(uploads)]
uploads.append(body)
response = {'valid': {'item_identity': identity}, 'absent': {'status': 'stored'},
'invalid': {'item_identity': 'bad'}, 'bad-json': 'bad-json', 'non-object': []}[mode]
return Response(response)
def get(self, url, **kw):
pid = url.rsplit('/', 1)[-1]
return Response({pid: {'status': {'completed': True, 'status_str': 'success'}}})
def delete(self, url, **kw):
runtime.delete_worker_item(url.rsplit('/', 1)[-1])
return Response({})
async def dispatch(worker, prompt, meta, **kwargs):
dispatched.append(copy.deepcopy(prompt))
delivery = prompt['end']['inputs']
token, index = delivery['worker_result_token'], delivery['worker_item_index']
identity = prompt['start']['inputs'].get('worker_item_identity')
options = {'item_identity': identity, 'item_index': index} if identity else {}
data = runtime.consume_worker_item(token, **options)
if identity:
runtime.finish_worker_item(token, identity, index)
runtime.set_master_result(context, index, token, {
'image': data['image'], 'text': str(index), 'values': {'value_3': index}})
assert kwargs['partial_execution_targets'] == ['end']
return f'worker-{index}'
dependencies = {
'.api.orchestration.dispatch': types.SimpleNamespace(
preflight_image_list_workers=preflight, dispatch_worker_prompt=dispatch),
'.api.orchestration.prompt_transform': types.SimpleNamespace(
_resolve_direct_workers_for_image_list=lambda *args: workers),
'.api.orchestration.media_sync': types.SimpleNamespace(sync_worker_media=sync_media),
'.utils.config': types.SimpleNamespace(load_config=lambda: {'settings': {}},
get_worker_timeout_seconds=lambda: 60),
'.utils.network': types.SimpleNamespace(build_master_callback_url=lambda *a, **kw: 'http://master',
build_worker_url=lambda worker, path: 'http://worker' + path,
get_client_session=AsyncMock(return_value=Session())),
'.utils.logging': types.SimpleNamespace(log=lambda *args: None),
}
for suffix, module in dependencies.items():
monkeypatch.setitem(sys.modules, name + suffix, module)
monkeypatch.setitem(sys.modules, 'aiohttp', types.SimpleNamespace(
ClientTimeout=lambda **kwargs: kwargs, ClientError=ConnectionError))
comfy = types.ModuleType('comfy')
comfy.model_management = types.SimpleNamespace(processing_interrupted=lambda: False)
monkeypatch.setitem(sys.modules, 'comfy', comfy)
monkeypatch.setitem(sys.modules, 'comfy.model_management', comfy.model_management)
items = [item(), item()]
items[1]['index'] = 1
context = runtime.create_master_job('start', 'end', graph(), items, ['worker'], '')
state = runtime.get_master_job(context)
try:
result = asyncio.run(runtime._run_image_list_job(state, include_values=True))
assert result[2] == ['0', '1']
assert [row['value_3'] for row in result[3]] == [0, 1]
assert len(uploads) == len(dispatched) == 2
assert sync_media.await_count == 1
for index, prompt in enumerate(dispatched):
start_inputs = prompt['start']['inputs']
if responses[index] == 'valid':
assert start_inputs == {'worker_item_identity': runtime.item_content_identity(items[index])}
assert prompt['end']['inputs']['worker_context_id'] == context
else:
assert 'worker_item_identity' not in start_inputs
assert start_inputs['worker_item_token'] == uploads[index]['token']
assert start_inputs['worker_context_id'] == context
assert 'worker_item_identity' not in prompt['end']['inputs']
assert prompt['end']['inputs']['worker_result_token'] == uploads[index]['token']
assert not runtime._WORKER_ITEMS
finally:
runtime.remove_master_job(context)
runtime._WORKER_ITEMS.clear()
def test_runtime_local_path_hashes_without_transport(monkeypatch):
name = '_native_cache_identity_local_runtime'
for suffix in ('', '.image_list', '.api', '.api.orchestration', '.utils'):
module = types.ModuleType(name + suffix)
module.__path__ = [str(ROOT / suffix.lstrip('.').replace('.', '/'))]
monkeypatch.setitem(sys.modules, name + suffix, module)
spec = importlib.util.spec_from_file_location(name + '.image_list.runtime', ROOT / 'image_list/runtime.py')
runtime = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, spec.name, runtime)
spec.loader.exec_module(runtime)
dispatched = []
class Slot:
def __init__(self, **kw):
pass
def submit(self, prompt, prompt_id, end_id, workflow_meta):
dispatched.append(prompt)
delivery = prompt[end_id]['inputs']
identity = prompt['start']['inputs']['worker_item_identity']
data = runtime.consume_worker_item(delivery['worker_result_token'], item_identity=identity,
item_index=delivery['worker_item_index'])
runtime.finish_worker_item(delivery['worker_result_token'], identity, data['index'])
runtime.set_master_result(delivery['worker_context_id'], data['index'], delivery['worker_result_token'],
{'image': data['image'], 'values': {'value_3': data['index']}})
future = asyncio.get_running_loop().create_future()
future.set_result(None)
return future
def request_stop(self):
pass
async def close(self):
pass
def forbidden(*args, **kwargs):
raise AssertionError('local path attempted HTTP/transport')
dependencies = {
'.image_list.local_executor': types.SimpleNamespace(LocalExecutionSlot=Slot),
'.api.orchestration.dispatch': types.SimpleNamespace(
preflight_image_list_workers=forbidden, dispatch_worker_prompt=forbidden),
'.api.orchestration.prompt_transform': types.SimpleNamespace(
_resolve_direct_workers_for_image_list=lambda *args: []),
'.utils.config': types.SimpleNamespace(load_config=lambda: {'settings': {}},
get_worker_timeout_seconds=lambda: 60),
'.utils.network': types.SimpleNamespace(build_master_callback_url=forbidden,
build_worker_url=forbidden, get_client_session=forbidden),
'.utils.logging': types.SimpleNamespace(log=lambda *args: None),
}
for suffix, module in dependencies.items():
monkeypatch.setitem(sys.modules, name + suffix, module)
monkeypatch.setitem(sys.modules, 'aiohttp', types.SimpleNamespace(
ClientTimeout=lambda **kwargs: kwargs, ClientError=ConnectionError))
comfy = types.ModuleType('comfy')
comfy.model_management = types.SimpleNamespace(processing_interrupted=lambda: False)
monkeypatch.setitem(sys.modules, 'comfy', comfy)
monkeypatch.setitem(sys.modules, 'comfy.model_management', comfy.model_management)
monkeypatch.setattr(runtime, 'MAX_PAYLOAD_BYTES', 1)
monkeypatch.setattr(runtime, 'encode_item', forbidden)
data = item()
context = runtime.create_master_job('start', 'end', graph(), [data], [], '', master_participates=True)
try:
result = asyncio.run(runtime._run_image_list_job(runtime.get_master_job(context), include_values=True))
assert dispatched[0]['start']['inputs'] == {'worker_item_identity': runtime.item_content_identity(data),
'image_list_local_execution': True}
assert result[3][0]['value_3'] == 0
assert not runtime._WORKER_ITEMS
finally:
runtime.remove_master_job(context)
runtime._WORKER_ITEMS.clear()
def test_real_native_signatures_and_cached_delivery():
core = Path(os.environ.get('COMFYUI_SOURCE_ROOT', str(Path(sys.executable).parents[2])))
if not (core / 'comfy_execution/caching.py').exists():
pytest.skip('real ComfyUI source not available beside this interpreter')
result = subprocess.run([sys.executable, str(ROOT / 'tests/helpers/image_list_native_cache_identity.py'), str(core)],
text=True, capture_output=True, timeout=90)
assert result.returncode == 0, result.stdout + result.stderr
assert 'NATIVE_CACHE_IDENTITY_OK' in result.stdout
+45
View File
@@ -345,6 +345,7 @@ class LifecycleTests(unittest.IsolatedAsyncioTestCase):
index = int(media["request_id"].rsplit(":", 1)[1])
data = item(index)
values = {"images": data["image"], "masks": data["mask"], "texts": str(index)}
values.update(getattr(self, "override_values", {}))
for n, binding in enumerate(media["outputs"]):
descriptor, payload = codec.encode_payload(binding["kind"], values[binding["socket"]])
aid = f"result:{n}"
@@ -671,6 +672,50 @@ class LifecycleTests(unittest.IsolatedAsyncioTestCase):
assert not self.server.distributed_pending_jobs
self.api.submit.assert_not_awaited()
async def test_sparse_overrides_overlay_master_values_without_forwarding(self):
from test_image_list_serverless_values import assert_exact
self.override_values = {"value_2": 2**63 + 333, "value_10": False}
self.state.original_prompt["ns/end"]["inputs"].update(
value_2=["process", 2], value_10=["process", 3])
slot, handle = await self.dispatch()
await self.finish(handle)
result = self.controller.poll({slot: handle})[2]
assert set(result["values"]) == set(self.override_values)
for name, expected in self.override_values.items():
assert_exact(result["values"][name], expected)
state = copy.copy(self.state)
state.items = [self.state.items[0]]
columns = runtime._collect_output_lists([result], state=state, include_values=True)
assert columns[3][0] == {**state.items[0]["values"], **self.override_values}
assert self.state.items[0]["values"]["value_2"] == 123
async def test_override_missing_extra_wrong_nonfinite_multiple_results_fail(self):
self.override_values = {"value_2": 42}
self.state.original_prompt["ns/end"]["inputs"]["value_2"] = ["process", 2]
for index, bad in enumerate(("missing", "extra", "wrong", "nonfinite", "multiple", "none")):
slot, handle = await self.dispatch(index)
await self.finish(handle)
queue = self.server.distributed_pending_jobs[handle["job_id"]]
message = queue.get_nowait()
media = message["media"]
if bad == "missing":
media.pop("value_2")
elif bad == "extra":
media["value_3"] = [1]
else:
media["value_2"] = {"wrong": [{}], "nonfinite": [float("nan")],
"multiple": [1, 2], "none": [None]}[bad]
queue.put_nowait(message)
with pytest.raises((ValueError, TypeError, RuntimeError)):
self.controller.poll({slot: handle})
async def test_no_override_results_preserve_original_return_shape(self):
slot, handle = await self.dispatch()
await self.finish(handle)
result = self.controller.poll({slot: handle})[2]
assert set(result) == {"image", "mask", "text"}
assert not any(binding["kind"] == "VALUE" for binding in self.jobs["provider-0"]["media"]["outputs"])
async def test_duplicate_index_and_mismatched_handle_fail(self):
slot, handle = await self.dispatch()
with pytest.raises(ValueError, match="duplicate"):
+165
View File
@@ -0,0 +1,165 @@
"""Sparse End overrides through the standalone serverless bridge, CPU only."""
import asyncio
import copy
import math
import os
from pathlib import Path
import struct
import subprocess
import sys
import pytest
import torch
from unittest.mock import patch
from test_image_list_serverless import adapter, codec, contract, handler, item, prepare, source
@pytest.mark.skipif(not os.environ.get('COMFYUI_SOURCE_ROOT'), reason='set COMFYUI_SOURCE_ROOT for real CPU executor')
def test_real_cpu_standalone_bridge_calculated_overrides(tmp_path):
env = dict(os.environ, CUDA_VISIBLE_DEVICES='', HIP_VISIBLE_DEVICES='', ROCR_VISIBLE_DEVICES='',
PYTHONDONTWRITEBYTECODE='1')
env.pop('PYTHONPATH', None)
helper = Path(__file__).parent / 'helpers' / 'image_list_serverless_values_check.py'
result = subprocess.run([sys.executable, str(helper), os.environ['COMFYUI_SOURCE_ROOT'], str(tmp_path)],
env=env, text=True, capture_output=True, timeout=90)
assert result.returncode == 0, result.stdout + result.stderr
assert 'IMAGE_LIST_SERVERLESS_VALUES_CPU_OK' in result.stdout
def assert_exact(actual, expected):
assert type(actual) is type(expected)
if isinstance(expected, torch.Tensor):
assert actual.dtype == expected.dtype and actual.shape == expected.shape
assert torch.equal(actual.contiguous().view(torch.uint8), expected.contiguous().view(torch.uint8))
elif isinstance(expected, float):
assert struct.pack('!d', actual) == struct.pack('!d', expected)
else:
assert actual == expected
@pytest.mark.parametrize('value', [False, True, 2**63 + 123, -(2**100), -0.0, 1.0000000000000002,
'', ' \r\n雪 ', torch.ones(2, 2, 3, 4, dtype=torch.float64),
torch.ones(2, 2, 3, dtype=torch.bfloat16)])
def test_value_codec_exact_types(value):
descriptor, payload = codec.encode_payload('VALUE', value)
assert_exact(codec.decode_payload('VALUE', descriptor, payload), value)
@pytest.mark.parametrize('value', [None, {}, [], (1,), object(), float('inf'), float('nan'),
torch.ones(2), torch.ones(1, 2, 3, dtype=torch.int64),
torch.full((1, 2, 3), float('nan'))])
def test_value_codec_rejects_unsupported_or_nonfinite(value):
with pytest.raises((ValueError, TypeError)):
codec.encode_payload('VALUE', value)
def test_sparse_exports_never_create_collector_or_forward_unchanged_values():
async def check():
prompt, value = source(), item()
prompt['ns/end']['inputs'].update(value_2=['ns/start', 5], value_10=['process', 2])
original = copy.deepcopy(prompt)
job = await prepare(prompt, value)
assert prompt == original
bindings = {binding['socket']: binding for binding in job['manifest']}
assert set(bindings) == {'images', 'masks', 'texts', 'value_2', 'value_10'}
assert bindings['value_2']['kind'] == bindings['value_10']['kind'] == 'VALUE'
assert all('Collector' not in node['class_type'] for node in job['workflow'].values())
assert all(node['class_type'] not in (adapter.runtime.START_CLASS, adapter.runtime.END_CLASS)
for node in job['workflow'].values())
handler._validate_request({key: job[key] for key in ('workflow', 'images', 'media')})
asyncio.run(check())
@pytest.mark.parametrize('slot', ['value_0', 'value_11', 'value_01', 'values.value_1'])
def test_unknown_end_override_slot_rejected(slot):
prompt = source()
prompt['ns/end']['inputs'][slot] = ['ns/start', 4]
with pytest.raises(ValueError, match='slot'):
asyncio.run(prepare(prompt))
def test_value_codec_budgets_and_malformed_payload():
with pytest.raises(ValueError, match='byte limit'):
codec.encode_payload('VALUE', '雪', max_bytes=2)
descriptor, payload = codec.encode_payload('VALUE', True)
with pytest.raises(ValueError):
codec.decode_payload('VALUE', dict(descriptor, type='INT'), payload)
descriptor, payload = codec.encode_payload('VALUE', -0.0)
with pytest.raises(ValueError):
codec.decode_payload('VALUE', descriptor, struct.pack('<d', math.inf))
with pytest.raises(ValueError):
codec.decode_payload('VALUE', descriptor, payload + b'0')
with pytest.raises(ValueError, match='byte limit'):
codec.decode_payload('VALUE', descriptor, payload, max_bytes=7)
@pytest.mark.parametrize('fmt', ['exact', 'legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1'])
def test_end_image_policy_never_compresses_override_tensors(fmt):
prompt = source()
prompt['ns/end']['inputs'].update(value_10=['ns/start', 0], transfer_format=fmt, transfer_quality=73)
job = asyncio.run(prepare(prompt))
for binding in job['manifest']:
controls = job['workflow'][binding['node_id']]['inputs']
if binding['socket'] == 'images' and fmt != 'exact':
assert controls['transfer_format'] == fmt and controls['transfer_quality'] == 73
else:
assert 'transfer_format' not in controls and 'transfer_quality' not in controls
default = asyncio.run(prepare())
assert all('transfer_format' not in n['inputs'] and 'transfer_quality' not in n['inputs']
for n in default['workflow'].values())
def value_response():
binding = dict(binding_id='binding', collector_id='end', job_id='job', participant_id='pool',
node_id='export', socket='value_10', kind='VALUE')
descriptor, payload = codec.encode_payload('VALUE', 2**63 + 7)
artifact = contract.artifact_from_payload('artifact', 'VALUE', descriptor, payload)
return binding, dict(version=1, profile=contract.PROFILE, request_id='request',
bindings=[dict(binding, count=1)], artifacts=[artifact],
items=[dict(binding_id='binding', item_index=0, artifact_id='artifact')])
@pytest.mark.parametrize('bad', ['missing', 'extra', 'wrong_kind', 'multiple', 'extra_artifact', 'nonfinite'])
def test_override_response_rejected_atomically(bad):
binding, response = value_response()
if bad == 'missing':
response['bindings'] = []
elif bad == 'extra':
response['bindings'].append(dict(binding, binding_id='other', socket='value_9', count=1))
elif bad == 'wrong_kind':
response['artifacts'][0]['kind'] = 'STRING'
elif bad == 'multiple':
response['bindings'][0]['count'] = 2
elif bad == 'extra_artifact':
response['artifacts'].append(dict(response['artifacts'][0], id='extra'))
else:
descriptor, _ = codec.encode_payload('VALUE', 1.0)
response['artifacts'][0] = contract.artifact_from_payload(
'artifact', 'VALUE', descriptor, struct.pack('<d', float('nan')))
with pytest.raises(ValueError):
asyncio.run(contract.decode_response(response, request_id='request', bindings=[binding]))
def test_override_request_and_result_budgets():
prompt = source()
prompt['ns/end']['inputs']['value_10'] = '雪' * 100
with patch.object(contract, 'MAX_ARTIFACT_BYTES', 128):
with pytest.raises(ValueError, match='byte limit'):
asyncio.run(prepare(prompt))
binding, response = value_response()
with pytest.raises(ValueError, match='total byte limit'):
asyncio.run(contract.decode_response(response, request_id='request', bindings=[binding], max_total_bytes=1))
def test_collector_compiler_keeps_only_its_public_sockets():
from importlib import import_module
compiler = import_module(adapter.__package__.rsplit('.', 1)[0] + '.api.orchestration.serverless_media_prompt')
assert compiler.COLLECTOR_SOCKETS == ('images', 'audio')
graph = {'source': {'class_type': 'Source', 'inputs': {}},
'collector': {'class_type': 'DistributedCollector', 'inputs': {
'images': ['source', 0], 'value_10': ['source', 1], 'masks': ['source', 2],
'multi_job_id': 'job', 'worker_id': 'pool'}}}
_, bindings = compiler.compile_media_prompt(graph, 'request')
assert [binding['socket'] for binding in bindings] == ['images']
+51 -4
View File
@@ -147,6 +147,36 @@ def test_compressed_request_combined_budget(monkeypatch):
rt.decode_result(encoded)
@pytest.mark.parametrize("fmt", FORMATS)
def test_return_formats_preserve_exact_sparse_override_values(fmt):
from test_image_list_values import assert_values, values
source = result()
source["values"] = values()
source["values"]["value_10"] = source["values"].pop("value_6")
encoded = rt.encode_result(source, transfer_format=fmt, transfer_quality=73)
assert set(encoded["values"]) == set(source["values"])
for name, value in source["values"].items():
assert encoded["values"][name] == rt.encode_value(value, rt.value_kind(value, name))
decoded = rt.decode_result(json.loads(rt.encode_request_payload(encoded)))
assert_values(decoded["values"], source["values"])
assert torch.equal(decoded["mask"], source["mask"])
assert decoded["text"] == source["text"]
def test_compressed_aggregate_budget_includes_override_values(monkeypatch):
source = result()
encoded_without_values = rt.encode_result(source, transfer_format="png")
source["values"] = {"value_10": "x" * 500}
encoded = rt.encode_result(source, transfer_format="png")
monkeypatch.setattr(rt, "MAX_REQUEST_BYTES", len(rt.encode_request_payload(encoded_without_values)) + 100)
with pytest.raises(ValueError, match="request"):
rt.encode_result(source, transfer_format="png")
with patch.object(image_codec, "decode_image", side_effect=AssertionError("oversize reached codec")):
with pytest.raises(ValueError, match="request"):
rt.decode_result(encoded)
@pytest.fixture
def dispatch_module(monkeypatch):
name = "_image_list_transfer_tests"
@@ -223,11 +253,15 @@ def test_capability_endpoint_advertises_shared_formats(monkeypatch):
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
response = asyncio.run(module.image_list_capabilities_endpoint(None))
assert json.loads(response.text)["image_transfer_formats"] == list(FORMATS)
caps = json.loads(response.text)
assert caps["image_transfer_formats"] == list(FORMATS)
assert all(caps[name] is True for name in ("supported", "execution_targets", "value_results"))
assert "DistributedImageListGetValue" in caps["node_ids"]
@pytest.mark.parametrize("reject_call", [None, 1, 2, 3])
def test_runtime_initial_and_each_nonexact_assignment_preflight(reject_call):
@pytest.mark.parametrize("with_values", [False, True])
def test_runtime_initial_and_each_nonexact_assignment_preflight(reject_call, with_values):
from test_image_list_runtime_boundaries import RuntimeBoundaryTests
async def exercise():
@@ -237,11 +271,22 @@ def test_runtime_initial_and_each_nonexact_assignment_preflight(reject_call):
context = fixture.create(2)
state = fixture.rt.get_master_job(context)
state.original_prompt[state.end_id]["inputs"].update(transfer_format="png", transfer_quality=87)
if with_values:
state.original_prompt[state.end_id]["inputs"]["value_10"] = [state.start_id, 2]
for item in state.items:
item["values"] = {"value_1": "retained"}
original_set = fixture.rt.set_master_result
def set_with_override(context, index, token, result):
return original_set(context, index, token, {**result, "values": {"value_10": index}})
fixture.rt.set_master_result = set_with_override
calls = []
async def preflight(workers, **kwargs):
calls.append(kwargs)
assert kwargs["required_transfer_formats"] == ("png",)
assert kwargs.get("require_value_results", False) is with_values
if len(calls) == reject_call:
raise RuntimeError("upgrade worker; no work was submitted")
return {"worker": True}
@@ -249,13 +294,15 @@ def test_runtime_initial_and_each_nonexact_assignment_preflight(reject_call):
sys.modules["_image_list_boundaries.api.orchestration.dispatch"].preflight_image_list_workers = preflight
if reject_call:
with pytest.raises(RuntimeError, match="upgrade"):
await fixture.rt.run_image_list_job(context)
await fixture.rt.run_image_list_job(context, include_values=with_values)
assert len(fixture.uploads) == max(0, reject_call - 2)
assert len(fixture.prompts) == max(0, reject_call - 2)
else:
await fixture.rt.run_image_list_job(context)
result_lists = await fixture.rt.run_image_list_job(context, include_values=with_values)
assert len(calls) == 3
assert len(fixture.uploads) == 2
if with_values:
assert result_lists[3] == [{"value_1": "retained", "value_10": index} for index in range(2)]
assert not fixture.rt._MASTER_JOBS
finally:
# Do not call doCleanups(): this borrowed IsolatedAsyncioTestCase has
+165
View File
@@ -0,0 +1,165 @@
"""Context-carried values and sparse, per-item End overrides."""
import copy
from unittest.mock import patch
import pytest
import torch
from image_list import runtime as rt
from image_list.execution_plan import prepare_image_list_execution
def value_graph():
# Use a minimal, closed region with an override-only ancestor.
return {
'source': {'class_type': 'Source', 'inputs': {}},
'start': {'class_type': rt.START_CLASS, 'inputs': {'images': ['source', 0]}},
'override': {'class_type': 'Transform', 'inputs': {'value': ['start', 4]}},
'end': {'class_type': rt.END_CLASS, 'inputs': {
'images': ['start', 0], 'context': ['start', 3], 'value_1': ['override', 0],
'value_10': ['start', 5]}},
}
def values():
return {'value_1': torch.arange(72, dtype=torch.float64).reshape(2, 3, 4, 3),
'value_2': torch.ones(3, 2, 2, dtype=torch.bfloat16), 'value_3': 'name',
'value_4': 42, 'value_5': 0.125, 'value_6': True}
def assert_values(actual, expected):
assert actual.keys() == expected.keys()
for key, value in expected.items():
assert type(actual[key]) is type(value)
if isinstance(value, torch.Tensor):
assert actual[key].dtype == value.dtype
assert torch.equal(actual[key], value)
else:
assert actual[key] == value
def test_normalize_and_roundtrip_all_override_kinds_keeps_batches_whole():
expected = values()
result = rt.normalize_worker_result(torch.zeros(1, 2, 2, 3),
values={key: [value] for key, value in expected.items()})
assert_values(result['values'], expected)
wire = rt.encode_result(result)
assert_values(rt.decode_result(wire)['values'], expected)
assert wire['values']['value_1']['shape'][0] == 2
assert wire['values']['value_2']['shape'][0] == 3
@pytest.mark.parametrize('bad', [None, [], [None], [1, 2], [[1]], object(),
float('nan'), float('inf'), torch.tensor([float('nan')]),
torch.tensor([float('inf')]), torch.ones(1, dtype=torch.int32)])
def test_invalid_override_rejected(bad):
with pytest.raises((ValueError, TypeError)):
rt.normalize_worker_result(torch.zeros(1, 2, 2, 3), values={'value_1': bad})
def test_unknown_override_names_and_wire_types_rejected():
for name in ('value_0', 'value_11', 'values.value_1'):
with pytest.raises(ValueError, match='slot'):
rt.normalize_worker_result(torch.zeros(1, 2, 2, 3), values={name: 1})
base = rt.encode_result({'image': torch.zeros(1, 2, 2, 3)})
for payload in (None, [], {'value_0': {'type': 'INT', 'value': 1}},
{'value_1': {'type': 'LATENT', 'value': {}}},
{'value_1': {'type': 'FLOAT', 'value': float('nan')}},
{'value_1': {'type': 'STRING', 'value': None}},
{'value_1': rt.encode_value(torch.tensor([float('inf')]), 'TENSOR')}):
with pytest.raises((ValueError, TypeError)):
rt.decode_result({**base, 'values': payload})
def test_no_overrides_keeps_legacy_wire_and_does_not_encode_original_tensors():
image = torch.zeros(1, 2, 2, 3)
result = rt.normalize_worker_result(image)
assert set(result) == {'image', 'mask', 'text'}
with patch.object(rt, 'encode_value', wraps=rt.encode_value) as encode:
wire = rt.encode_result(result)
assert set(wire) == {'image', 'mask', 'text'}
assert [call.args[1] for call in encode.call_args_list] == ['IMAGE', 'MASK', 'STRING']
assert set(rt.decode_result(wire)) == {'image', 'mask', 'text'}
def test_override_ancestors_survive_plan_and_worker_prompt():
prompt = value_graph()
frozen = copy.deepcopy(prompt)
prepared, targets = prepare_image_list_execution(prompt)
assert prepared == frozen and prompt == frozen
assert targets == ['end']
region = rt.validate_image_list_regions(prepared)[0]
assert 'override' in region['branch_ids']
worker = rt.build_worker_prompt(prepared, region, 'token', 'context', 0, 'http://master')
assert worker['end']['inputs']['value_1'] == ['override', 0]
assert worker['end']['inputs']['value_10'] == ['start', 5]
assert 'source' not in worker
for bad in ('value_0', 'value_11', 'values.value_1'):
invalid = copy.deepcopy(prompt)
invalid['end']['inputs'][bad] = ['start', 4]
with pytest.raises(ValueError, match='slot'):
rt.validate_image_list_regions(invalid)
def test_missing_or_unexpected_result_override_fails_before_recording():
prompt = value_graph()
items = rt.normalize_image_list_inputs(torch.zeros(1, 2, 2, 3), values={'value_1': 'original'})
context = rt.create_master_job('start', 'end', prompt, items, ['worker'], '')
try:
rt.register_assignment(context, 0, 'worker', 'token')
for result_values in ({}, {'value_1': 'updated'},
{'value_1': 'updated', 'value_10': 10, 'value_9': False}):
with pytest.raises(ValueError, match='override'):
rt.set_master_result(context, 0, 'token', {'image': items[0]['image'], 'values': result_values})
assert rt.get_master_job(context).received_indexes == set()
rt.set_master_result(context, 0, 'token', {'image': items[0]['image'],
'values': {'value_1': 'updated', 'value_10': 10}})
assert rt.get_master_job(context).received_indexes == {0}
finally:
rt.remove_master_job(context)
def test_override_return_request_budget_is_aggregate():
result = rt.normalize_worker_result(torch.zeros(1, 2, 2, 3), values={'value_1': ['x' * 500]})
with patch.object(rt, 'MAX_REQUEST_BYTES', 400):
with pytest.raises(ValueError, match='request'):
rt.encode_request_payload({'result': rt.encode_result(result)})
class TestResultHTTP:
def test_missing_and_invalid_overrides_fail_through_real_http(self):
import asyncio
from aiohttp import web
from aiohttp.test_utils import TestClient, TestServer
from test_image_list_integration_gaps import load_routes
async def check():
_module, routes = load_routes()
app = web.Application()
app.add_routes(routes)
prompt = value_graph()
items = rt.normalize_image_list_inputs(torch.zeros(1, 2, 2, 3), values={'value_1': 'original'})
context = rt.create_master_job('start', 'end', prompt, items, ['worker'], '')
token = rt.create_worker_task_token()
rt.register_assignment(context, 0, 'worker', token)
try:
async with TestClient(TestServer(app)) as client:
caps = await (await client.get('/distributed/image-list/capabilities')).json()
assert caps['value_results'] is True and caps['protocol_version'] == 2
result = rt.encode_result({'image': items[0]['image']})
envelope = {'context_id': context, 'token': token, 'item_index': 0, 'result': result}
response = await client.post('/distributed/image-list/result', json=envelope)
assert response.status == 400
assert 'missing' in (await response.json())['error']
assert not rt.get_master_job(context).received_indexes
result['values'] = {'value_1': {'type': 'INT', 'value': [1, 2]},
'value_10': {'type': 'INT', 'value': 10}}
response = await client.post('/distributed/image-list/result', json=envelope)
assert response.status == 400
assert not rt.get_master_job(context).received_indexes
result['values']['value_1'] = {'type': 'STRING', 'value': 'new'}
response = await client.post('/distributed/image-list/result', json=envelope)
assert response.status == 200
assert rt.get_master_job(context).results[0]['values'] == {'value_1': 'new', 'value_10': 10}
finally:
rt.remove_master_job(context)
asyncio.run(check())
+2 -2
View File
@@ -94,7 +94,7 @@ async def check():
assert await extension.get_node_list() == [MediaImportV1, MediaExportV1]
imp = MediaImportV1.GET_NODE_INFO_V1()
exp = MediaExportV1.GET_NODE_INFO_V1()
assert imp['output'] == ['IMAGE','AUDIO','MASK','STRING','VIDEO'], imp
assert imp['output'] == ['IMAGE','AUDIO','MASK','STRING','VIDEO','*'], imp
assert exp['output_node'] and MediaExportV1.define_schema().is_input_list, exp
with tempfile.TemporaryDirectory(prefix='typed-media-', dir=os.environ.get('TMPDIR')) as scratch:
base = Path(scratch)
@@ -152,7 +152,7 @@ async def check():
mapped, missing, hidden = execution.get_input_data({'artifact_json':json.dumps(artifact),'filename':filename},MediaImportV1,'import')
got = await execution._async_map_node_over_list('probe','import',MediaImportV1,mapped,MediaImportV1.FUNCTION,v3_data=hidden)
got = await execution.resolve_map_node_over_list_results(got)
assert len(got[0].result) == 5
assert len(got[0].result) == 6
assert all(x is None for n,x in enumerate(got[0].result) if n != list(SOCKET_KINDS).index(socket))
bad = dict(artifact,sha256='0'*64)
try: import_value(bad,filename)
-314
View File
@@ -1,314 +0,0 @@
"""CPU-only typed collection and optional native V3 mapping acceptance."""
import asyncio
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import types
import pytest
import torch
ROOT = Path(__file__).resolve().parents[1]
SOCKETS = ('images', 'audio', 'masks', 'texts', 'video')
WORKERS = ['serverless:pool:1', 'serverless:pool:2']
class Manager:
def __init__(self, *, pending=False, owned=True):
self.pending, self.owned = pending, owned
self.cancelled = []
def owns(self, job, worker):
return self.owned
def is_pending(self, job, worker):
return self.pending
async def cancel(self, **kwargs):
self.cancelled.append(kwargs)
@pytest.fixture
def module(monkeypatch):
package = '_media_collector_test'
for name, path in ((package, ROOT), (package + '.nodes', ROOT / 'nodes'),
(package + '.utils', ROOT / 'utils'),
(package + '.serverless_bridge', ROOT / 'serverless_bridge')):
stub = types.ModuleType(name)
stub.__path__ = [str(path)]
monkeypatch.setitem(sys.modules, name, stub)
io = types.SimpleNamespace(ComfyNode=object, NodeOutput=lambda *items: types.SimpleNamespace(result=items))
api = types.ModuleType('comfy_api.v0_0_2')
api.io = io
monkeypatch.setitem(sys.modules, 'comfy_api', types.ModuleType('comfy_api'))
monkeypatch.setitem(sys.modules, 'comfy_api.v0_0_2', api)
management = types.ModuleType('comfy.model_management')
management.throw_exception_if_processing_interrupted = lambda: None
comfy = types.ModuleType('comfy')
comfy.model_management = management
monkeypatch.setitem(sys.modules, 'comfy', comfy)
monkeypatch.setitem(sys.modules, 'comfy.model_management', management)
ps = types.SimpleNamespace(distributed_pending_jobs={}, distributed_jobs_lock=asyncio.Lock())
server = types.ModuleType('server')
server.PromptServer = types.SimpleNamespace(instance=ps)
monkeypatch.setitem(sys.modules, 'server', server)
helper = types.ModuleType(package + '.utils.async_helpers')
helper.run_async_in_server_loop = asyncio.run
monkeypatch.setitem(sys.modules, helper.__name__, helper)
spec = importlib.util.spec_from_file_location(package + '.nodes.media_collector', ROOT / 'nodes/media_collector.py')
loaded = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, spec.name, loaded)
spec.loader.exec_module(loaded)
monkeypatch.setattr(loaded, 'POLL_SECONDS', 0.005)
monkeypatch.setattr(loaded, 'START_GRACE_SECONDS', 0.025)
loaded.test_server = ps
loaded.test_management = management
return loaded
def native_media(offset=0):
return {
'images': [torch.full((1, 2 + offset, 3, 4), offset, dtype=torch.float64),
torch.zeros((2, 1, 1, 3), dtype=torch.float16)],
'audio': [{'waveform': torch.zeros((1, 1 + offset, 8), dtype=torch.float64),
'sample_rate': 44100 + offset}],
'masks': [torch.zeros((1, 2 + offset, 3), dtype=torch.float64)],
'texts': ['', ' café\n🎥\r\n'],
'video': [object()],
}
def setup(module, messages=(), *, manager=None):
queue = asyncio.Queue()
for message in messages:
queue.put_nowait(message)
ps = module.test_server
ps.distributed_pending_jobs = {'job': queue, 'unrelated': asyncio.Queue()}
ps.distributed_serverless = manager if manager is not None else Manager()
return ps, queue
def message(worker, media):
return {'worker_id': worker, 'media': media, 'is_last': True}
def collect(module, **kwargs):
return module.DistributedMediaCollector.execute(**kwargs).result
def remote(module, **kwargs):
controls = dict(multi_job_id=['job'], enabled_worker_ids=[json.dumps(WORKERS)],
expected_media=[json.dumps(['texts'])], delegate_only=[True])
controls.update(kwargs)
return collect(module, **controls)
@pytest.mark.parametrize('controls', [{}, {'pass_through': [True]}, {'enabled_worker_ids': ['[]']}])
def test_all_five_passthrough_preserves_lists_and_native_values(module, controls):
values = native_media()
result = collect(module, **values, **controls)
assert len(result) == 5
for index, socket in enumerate(SOCKETS):
assert result[index] is values[socket]
def test_reversed_arrivals_master_then_workers_then_items(module):
master, first, second = native_media(), native_media(1), native_media(2)
ps, _ = setup(module, [message(WORKERS[1], second), message(WORKERS[0], first)])
result = remote(module, **master, delegate_only=[False], expected_media=[json.dumps(list(SOCKETS))])
for index, socket in enumerate(SOCKETS):
expected = master[socket] + first[socket] + second[socket]
assert len(result[index]) == len(expected)
assert all(actual is original for actual, original in zip(result[index], expected))
assert list(ps.distributed_pending_jobs) == ['unrelated']
assert ps.distributed_serverless.cancelled == []
def test_delegate_only_and_empty_string(module):
ps, _ = setup(module, [message(WORKERS[1], {'texts': ['b']}), message(WORKERS[0], {'texts': ['']})])
result = remote(module)
assert result == ([], [], [], ['', 'b'], [])
assert 'job' not in ps.distributed_pending_jobs
@pytest.mark.parametrize('kwargs', [{}, {'texts': []}, {'texts': ''}, {'texts': ['x'], 'is_worker': [True]},
{'texts': ['x'], 'enabled_worker_ids': ['["direct-worker"]']},
{'texts': ['x'], 'expected_media': ['["images"]']},
{'texts': ['x'], 'pass_through': [True, False]}, {'delegate_only': True},
{'texts': ['x'], 'enabled_worker_ids': ['["serverless:pool:1","serverless:pool:1"]']},
{'texts': ['x'], 'load_balance': ['false']}])
def test_rejects_missing_media_or_invalid_controls(module, kwargs):
with pytest.raises((ValueError, RuntimeError)):
collect(module, **kwargs)
@pytest.mark.parametrize('bad', [
{'worker_id': 'serverless:other:1', 'media': {'texts': ['x']}, 'is_last': True},
{'worker_id': WORKERS[0], 'media': {'texts': ['x']}, 'is_last': False},
{'worker_id': WORKERS[0], 'media': {'images': [object()]}, 'is_last': True},
{'worker_id': WORKERS[0], 'media': {'texts': []}, 'is_last': True},
{'worker_id': WORKERS[0], 'media': {'texts': 'x'}, 'is_last': True},
{'worker_id': WORKERS[0], 'media': {'texts': [None]}, 'is_last': True},
{'worker_id': WORKERS[0], 'media': {'texts': ['x'] * 65}, 'is_last': True},
{'worker_id': WORKERS[0], 'media': {'texts': ['x'], 'unknown': ['x']}, 'is_last': True},
{'worker_id': WORKERS[0], 'serverless_error': 'sanitized failure', 'is_last': True},
])
def test_rejects_bad_delivery_without_partial_output_and_cleans(module, bad):
ps, _ = setup(module, [bad])
with pytest.raises((ValueError, RuntimeError)):
remote(module)
assert list(ps.distributed_pending_jobs) == ['unrelated']
assert ps.distributed_serverless.cancelled == [{'collector_job_id': 'job'}]
@pytest.mark.parametrize('after_complete', [False, True])
def test_duplicate_delivery_rejected_including_final_queue_drain(module, after_complete):
first = message(WORKERS[0], {'texts': ['a']})
second = message(WORKERS[1], {'texts': ['b']})
messages = [first, second, first] if after_complete else [first, first, second]
ps, _ = setup(module, messages)
with pytest.raises(RuntimeError, match='[Dd]uplicate'):
remote(module)
assert 'job' not in ps.distributed_pending_jobs
assert ps.distributed_serverless.cancelled == [{'collector_job_id': 'job'}]
@pytest.mark.parametrize('manager', [None, Manager(owned=False), Manager(pending=False)])
def test_missing_manager_or_finished_participant_fails_bounded(module, manager):
ps, _ = setup(module)
ps.distributed_serverless = manager
with pytest.raises(RuntimeError, match='[Ss]erverless'):
remote(module)
assert 'job' not in ps.distributed_pending_jobs
def test_control_error_also_cleans_exact_job(module):
ps, _ = setup(module)
with pytest.raises(ValueError):
remote(module, expected_media='["unknown"]')
assert list(ps.distributed_pending_jobs) == ['unrelated']
assert ps.distributed_serverless.cancelled == [{'collector_job_id': 'job'}]
def test_launch_race_and_pending_deadline_owned_by_manager(module, monkeypatch):
ps, queue = setup(module)
ps.distributed_serverless = None
async def runner(coro):
async def produce():
await asyncio.sleep(0.01)
ps.distributed_serverless = Manager(pending=True)
await asyncio.sleep(0.035)
queue.put_nowait(message(WORKERS[1], {'texts': ['b']}))
queue.put_nowait(message(WORKERS[0], {'texts': ['a']}))
task = asyncio.create_task(produce())
result = await coro
await task
return result
monkeypatch.setattr(module, 'run_async_in_server_loop', lambda coro: asyncio.run(runner(coro)))
assert remote(module)[3] == ['a', 'b']
assert ps.distributed_serverless.cancelled == []
@pytest.mark.parametrize('cancel_task', [False, True])
def test_cancellation_while_blocked_cleans_and_propagates(module, monkeypatch, cancel_task):
ps, _ = setup(module, manager=Manager(pending=True))
class Interrupted(Exception):
pass
async def runner(coro):
task = asyncio.create_task(coro)
await asyncio.sleep(0.012)
if cancel_task:
task.cancel()
else:
def interrupt():
raise Interrupted('stop')
module.test_management.throw_exception_if_processing_interrupted = interrupt
return await task
monkeypatch.setattr(module, 'run_async_in_server_loop', lambda coro: asyncio.run(runner(coro)))
with pytest.raises(asyncio.CancelledError if cancel_task else Interrupted):
remote(module)
assert list(ps.distributed_pending_jobs) == ['unrelated']
assert ps.distributed_serverless.cancelled == [{'collector_job_id': 'job'}]
def test_replaced_queue_not_removed(module, monkeypatch):
ps, _ = setup(module, manager=Manager(pending=True))
replacement = asyncio.Queue()
async def runner(coro):
task = asyncio.create_task(coro)
await asyncio.sleep(0.012)
ps.distributed_pending_jobs['job'] = replacement
return await task
monkeypatch.setattr(module, 'run_async_in_server_loop', lambda coro: asyncio.run(runner(coro)))
with pytest.raises(RuntimeError, match='queue'):
remote(module)
assert ps.distributed_pending_jobs['job'] is replacement
def test_native_v3_schema_and_list_mapping_cpu():
core = os.environ.get('COMFYUI_SOURCE_ROOT')
if not core:
pytest.skip('Set COMFYUI_SOURCE_ROOT for CPU-only native V3 mapping')
probe = r'''
import asyncio, importlib.util, sys, types
from pathlib import Path
sys.path.insert(0, sys.argv[1])
import comfy.cli_args
comfy.cli_args.args.cpu = True
comfy.cli_args.args.disable_assets = True
import execution
import torch
import server
# No HTTP listener or PromptServer bootstrap is needed for schema/list mapping.
server.PromptServer.instance = types.SimpleNamespace()
root = Path(sys.argv[2])
for name, path in [('typed_collector', root), ('typed_collector.nodes', root/'nodes'),
('typed_collector.utils', root/'utils'),
('typed_collector.serverless_bridge', root/'serverless_bridge')]:
package = types.ModuleType(name)
package.__path__ = [str(path)]
sys.modules[name] = package
spec = importlib.util.spec_from_file_location('typed_collector.nodes.media_collector', root/'nodes/media_collector.py')
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
cls = module.DistributedMediaCollector
info = cls.GET_NODE_INFO_V1()
assert info['output'] == ['IMAGE','AUDIO','MASK','STRING','VIDEO'], info
assert info['output_is_list'] == [True]*5, info
assert cls.define_schema().is_input_list and cls.define_schema().accept_all_inputs
for name, kind in zip(['images','audio','masks','texts','video'], info['output']):
field = info['input']['optional'][name]
assert field[0] == kind and field[1]['forceInput'] is True, field
assert info['input']['required']['load_balance'][1]['default'] is False
async def check():
values = {'load_balance':False, 'pass_through':True}
prepared, missing, hidden = execution.get_input_data(values, cls, unique_id='probe')
assert not missing, missing
image = torch.zeros((1,2,3,4), dtype=torch.float64)
audio = {'waveform':torch.zeros((1,2,7)), 'sample_rate':48000}
mask = torch.ones((1,2,3), dtype=torch.float64)
from comfy_api.input_impl import VideoFromComponents
from comfy_api.util import VideoComponents
from fractions import Fraction
video = VideoFromComponents(VideoComponents(images=image[:,:,:,:3].float(), frame_rate=Fraction(24,1)))
native = {'images':[image], 'audio':[audio], 'masks':[mask], 'texts':[''], 'video':[video]}
prepared.update(native)
results = await execution._async_map_node_over_list('cpu','probe',cls,prepared,cls.FUNCTION,v3_data=hidden)
results = await execution.resolve_map_node_over_list_results(results)
assert len(results) == 1
for index, name in enumerate(native):
assert results[0].result[index] is native[name]
assert results[0].result[4][0] is video
print('MEDIA_COLLECTOR_CPU_OK')
asyncio.run(check())
'''
env = {**os.environ, 'CUDA_VISIBLE_DEVICES': '', 'HIP_VISIBLE_DEVICES': '', 'PYTHONDONTWRITEBYTECODE': '1'}
env.pop('PYTHONPATH', None)
result = subprocess.run([sys.executable, '-c', probe, core, str(ROOT)], env=env,
capture_output=True, text=True, timeout=90)
assert result.returncode == 0, result.stdout + result.stderr
assert 'MEDIA_COLLECTOR_CPU_OK' in result.stdout
+2 -1
View File
@@ -1,4 +1,4 @@
"""Opt-in, isolated CPU acceptance of the complete typed-media data path.
"""Opt-in CPU acceptance of Collector IMAGE/AUDIO and the five-kind bridge.
COMFYUI_SOURCE_ROOT must name an existing compatible checkout; run with that
checkout's Python environment. No installation, normal instance, or provider is
@@ -42,6 +42,7 @@ def test_typed_media_end_to_end_cpu():
assert result.returncode == 0, result.stdout + result.stderr
for marker in ("NATIVE_INPUTS_AND_DIRECT_EXPORT_REFS_OK", "REAL_HTTP_BRIDGE_RESULTS_OK",
"MASTER_AND_DELEGATE_ORDER_OK", "ATOMIC_DECODE_REJECTION_OK",
"IMAGE_TRANSFER_HTTP_OK",
"MEDIA_END_TO_END_CPU_OK"):
assert marker in result.stdout, result.stdout + result.stderr
print(result.stdout)
+24 -30
View File
@@ -1,4 +1,4 @@
"""Pure integration of participant identity, typed compilation and delegate pruning."""
"""Participant identity, typed compilation and legacy Collector pruning."""
import copy
import importlib
import json
@@ -16,17 +16,17 @@ for suffix in ("", ".api", ".api.orchestration", ".utils", ".serverless_bridge")
sys.modules[module.__name__] = module
pt = importlib.import_module(PACKAGE + ".api.orchestration.prompt_transform")
compiler = importlib.import_module(PACKAGE + ".api.orchestration.serverless_media_prompt")
contract = importlib.import_module(PACKAGE + ".serverless_bridge.contract")
SOCKETS = ("images", "audio")
def prompt():
def prompt(sockets=SOCKETS):
nodes = {
"source": {"class_type": "MediaSource", "inputs": {}},
"collector": {"class_type": "DistributedMediaCollector", "inputs": {
socket: ["source", index] for index, socket in enumerate(contract.SOCKET_KINDS)}},
"collector": {"class_type": "DistributedCollector", "inputs": {
socket: ["source", SOCKETS.index(socket)] for socket in sockets}},
}
for index, socket in enumerate(contract.SOCKET_KINDS):
nodes["sink:" + socket] = {"class_type": "Capture", "inputs": {"value": ["collector", index]}}
for socket in sockets:
nodes["sink:" + socket] = {"class_type": "Capture", "inputs": {"value": ["collector", SOCKETS.index(socket)]}}
return nodes
@@ -38,47 +38,41 @@ def prepare(graph, participant, delegate):
["serverless:pool:1", "serverless:pool:2"], jobs, "http://master", delegate, index)
def test_typed_worker_preparation_compiles_all_bindings_without_source_mutation():
def test_typed_worker_preparation_compiles_both_bindings_without_source_mutation():
graph = prompt()
before = copy.deepcopy(graph)
worker = prepare(graph, "serverless:pool:2", True)
compiled, bindings = compiler.compile_media_prompt(worker, "request:2")
assert len(bindings) == 5
assert len(bindings) == 2
assert {row["participant_id"] for row in bindings} == {"serverless:pool:2"}
assert {row["job_id"] for row in bindings} == {"run_collector"}
assert {row["socket"] for row in bindings} == set(contract.SOCKET_KINDS)
assert {row["socket"] for row in bindings} == set(SOCKETS)
assert set(compiled) == {"source"} | {row["node_id"] for row in bindings}
assert graph == before
def test_typed_delegate_master_keeps_outputs_without_sources_or_placeholders():
graph = prompt()
@pytest.mark.parametrize("sockets", [("images",), ("audio",), SOCKETS])
def test_delegate_master_keeps_collector_socket_contract_and_legacy_placeholder(sockets):
graph = prompt(sockets)
prepared = prepare(graph, "master", True)
before = copy.deepcopy(prepared)
delegated = pt.prepare_delegate_master_prompt(prepared, ["collector"])
assert set(delegated) == {"collector"} | {"sink:" + s for s in contract.SOCKET_KINDS}
assert "source" not in delegated
assert {"collector"} | {"sink:" + s for s in sockets} <= set(delegated)
inputs = delegated["collector"]["inputs"]
assert not set(contract.SOCKET_KINDS) & set(inputs)
assert json.loads(inputs["expected_media"]) == list(contract.SOCKET_KINDS)
assert "audio" not in inputs and "expected_media" not in inputs
placeholders = [key for key, node in delegated.items() if node['class_type'] == 'DistributedEmptyImage']
if "images" in sockets:
assert len(placeholders) == 1
assert inputs['images'] == [placeholders[0], 0]
else:
assert not placeholders and 'images' not in inputs
assert inputs["delegate_only"] is True
assert json.loads(inputs["enabled_worker_ids"]) == ["serverless:pool:1", "serverless:pool:2"]
assert prepared == before
def test_text_source_is_not_retained_as_lightweight_delegate_scalar(monkeypatch):
graph = {
"text": {"class_type": "PrimitiveString", "inputs": {"value": "local text"}},
"collector": {"class_type": "DistributedMediaCollector", "inputs": {"texts": ["text", 0]}},
"sink": {"class_type": "Capture", "inputs": {"value": ["collector", 3]}},
}
monkeypatch.setattr(pt, "_delegate_master_input_is_safe_scalar", lambda *args: True)
monkeypatch.setattr(pt, "_is_delegate_master_retained_upstream_node", lambda *args: True)
delegated = pt.prepare_delegate_master_prompt(prepare(graph, "master", True), ["collector"])
assert "text" not in delegated
assert "texts" not in delegated["collector"]["inputs"]
assert json.loads(delegated["collector"]["inputs"]["expected_media"]) == ["texts"]
@pytest.mark.parametrize("socket", contract.SOCKET_KINDS)
@pytest.mark.parametrize("socket", SOCKETS)
def test_participating_master_retains_exact_media_links(socket):
graph = prompt()
original = copy.deepcopy(graph)
+99 -78
View File
@@ -35,11 +35,11 @@ def node(class_type, **inputs):
return {"class_type": class_type, "inputs": inputs}
def text_prompt():
def collector_prompt():
return {
"text": node("PrimitiveString", value=" café雪\n"),
"collector": node("DistributedMediaCollector", texts=["text", 0], load_balance=False),
"save": node("TextSink", texts=["collector", 3]),
"source": node("EmptyImage", width=32, height=32, batch_size=1, color=0),
"collector": node("DistributedCollector", images=["source", 0], load_balance=False),
"save": node("SaveImage", images=["collector", 0], filename_prefix="local"),
}
@@ -95,7 +95,7 @@ def media_paths(tmp_path, monkeypatch):
def run(h, prompt=None, **kwargs):
return asyncio.run(h.q.orchestrate_distributed_execution(
prompt if prompt is not None else text_prompt(), None, "client", **kwargs))
prompt if prompt is not None else collector_prompt(), None, "client", **kwargs))
def assert_no_dispatch(h, *, no_probe=False):
@@ -117,10 +117,11 @@ def test_real_compile_pack_serialize_and_canonical_bindings(harness, media_paths
image_path = media_paths / "picture.png"
Image.new("RGBA", (2, 3), color=(17, 29, 41, 53)).save(image_path)
image_bytes = image_path.read_bytes()
prompt = text_prompt()
prompt = collector_prompt()
prompt["image"] = node("LoadImage", image="picture.png")
prompt["collector"]["inputs"].update(images=["image", 0], masks=["image", 1])
prompt["collector2"] = node("DistributedMediaCollector", texts=["text", 0])
prompt["audio"] = node("GeneratedAudio")
prompt["collector"]["inputs"].update(images=["image", 0], audio=["audio", 0])
prompt["collector2"] = node("DistributedCollector", audio=["audio", 0])
prompt["unused"] = node("PreviewImage", images=["image", 0])
source = copy.deepcopy(prompt)
with patch.object(h.compiler, "compile_media_prompt", wraps=h.compiler.compile_media_prompt) as compile_call, \
@@ -145,8 +146,8 @@ def test_real_compile_pack_serialize_and_canonical_bindings(harness, media_paths
assert media["outputs"] == job["manifest"]
assert media["outputs"] is not job["manifest"]
assert media["artifacts"] == media["inputs"] == []
assert {b["socket"] for b in job["manifest"]} == {"texts", "images", "masks"}
assert len(job["manifest"]) == 4
assert {b["socket"] for b in job["manifest"]} == {"images", "audio"}
assert len(job["manifest"]) == 3
assert base64.b64decode(job["images"][0]["image"]) == image_bytes
assert job["workflow"]["image"]["inputs"]["image"] == job["images"][0]["name"]
assert not ({"save", "unused", "collector", "collector2"} & set(job["workflow"]))
@@ -164,40 +165,50 @@ def test_real_compile_pack_serialize_and_canonical_bindings(harness, media_paths
assert json.loads(controls["binding_json"]) == binding
assert controls["request_id"] == media["request_id"]
assert master["collector"]["inputs"]["delegate_only"] is delegate
assert json.loads(master["collector"]["inputs"]["expected_media"]) == ["images", "masks", "texts"]
assert "expected_media" not in master["collector"]["inputs"]
if delegate:
assert "text" not in master and "image" not in master
assert not any(name in master["collector"]["inputs"] for name in ("images", "audio", "masks", "texts", "video"))
assert not any(n["class_type"] == "DistributedEmptyImage" for n in master.values())
assert not {"source", "audio"} & set(master)
# LoadImage is an existing explicitly retained lightweight source; the
# Collector is nevertheless detached from it and uses the placeholder.
assert master["image"] == prompt["image"]
assert "audio" not in master["collector"]["inputs"]
placeholder = master["collector"]["inputs"]["images"][0]
assert master[placeholder]["class_type"] == "DistributedEmptyImage"
else:
assert master["collector"]["inputs"]["texts"] == ["text", 0]
assert "text" in master and "image" in master
assert master["collector"]["inputs"]["audio"] == ["audio", 0]
assert "audio" in master and "image" in master
def test_delegate_disconnects_every_media_socket(harness):
prompt = text_prompt()
for index, name in enumerate(("images", "audio", "masks", "video")):
source_id = "source-" + name
prompt[source_id] = node("GeneratedMedia", seed=index)
prompt["collector"]["inputs"][name] = [source_id, 0]
@pytest.mark.parametrize("audio_only", [False, True])
def test_delegate_prunes_image_audio_sources_and_audio_only_needs_no_placeholder(harness, audio_only):
prompt = collector_prompt()
prompt["audio"] = node("GeneratedAudio")
prompt["collector"]["inputs"]["audio"] = ["audio", 0]
if audio_only:
del prompt["collector"]["inputs"]["images"]
prompt["save"] = node("AudioSink", audio=["collector", 1])
original = copy.deepcopy(prompt)
run(harness, prompt)
master = harness.q.queue_prompt_payload.call_args.args[0]
inputs = master["collector"]["inputs"]
sockets = {"images", "audio", "masks", "texts", "video"}
assert set(json.loads(inputs["expected_media"])) == sockets
assert not sockets.intersection(inputs)
assert set(master) == {"collector", "save"}
assert "audio" not in inputs and "expected_media" not in inputs
assert "source" not in master and "audio" not in master
if audio_only:
assert "images" not in inputs and set(master) == {"collector", "save"}
else:
assert master[inputs["images"][0]]["class_type"] == "DistributedEmptyImage"
for job in harness.manager.start.call_args.args[2]:
assert {b["socket"] for b in job["manifest"]} == sockets
assert {"text", "source-images", "source-audio", "source-masks", "source-video"} <= set(job["workflow"])
assert {b["socket"] for b in job["manifest"]} == ({"audio"} if audio_only else {"images", "audio"})
assert "audio" in job["workflow"]
assert ("source" in job["workflow"]) is not audio_only
assert harness.validator.validate_prepared_job(job) is True
assert prompt == original
@pytest.mark.parametrize("selected", ["master", "worker"])
def test_typed_load_balance_selects_master_or_one_serverless_slot(harness, selected):
h = harness
prompt = text_prompt()
prompt = collector_prompt()
prompt["collector"]["inputs"]["load_balance"] = "true"
h.q.select_least_busy_worker.side_effect = lambda candidates, **kw: next(
w for w in candidates if (w["id"] == "master") == (selected == "master"))
@@ -208,23 +219,24 @@ def test_typed_load_balance_selects_master_or_one_serverless_slot(harness, selec
assert result[2] == 0
h.manager.start.assert_not_awaited()
assert master["collector"]["inputs"]["delegate_only"] is False
assert "text" in master
assert "source" in master
else:
assert result[2] == 1
assert len(h.manager.start.call_args.args[2]) == 1
assert master["collector"]["inputs"]["delegate_only"] is True
assert "text" not in master
assert "source" not in master
@pytest.mark.parametrize("profile", [None, "legacy-image-v1", PROFILE])
def test_explicit_profile_compatibility_rejected_before_probe(harness, profile):
@pytest.mark.parametrize("profile", [None, "legacy-image-v1"])
def test_exact_rejects_legacy_pool_before_probe(harness, profile):
h = harness
prompt = legacy_prompt() if profile == PROFILE else text_prompt()
prompt = collector_prompt()
prompt["collector"]["inputs"]["transfer_format"] = "exact"
if profile is None:
h.pool.pop("media_profile")
else:
h.pool["media_profile"] = profile
with pytest.raises(ValueError, match="requires|require"):
with pytest.raises(ValueError, match="exact requires comfy-media-v1"):
run(h, prompt)
assert_no_dispatch(h, no_probe=True)
@@ -239,19 +251,26 @@ def test_invalid_explicit_profiles_fail_closed(harness, profile):
@pytest.mark.parametrize("direct_type", ["local", "remote", "cloud"])
@pytest.mark.parametrize("explicit", [False, True])
def test_direct_workers_never_silently_skipped_or_fallback(harness, direct_type, explicit):
def test_direct_workers_participate_alongside_typed_pools(harness, direct_type, explicit):
h = harness
h.config["workers"].append({"id": "direct", "type": direct_type, "host": "offline.invalid",
"enabled": not explicit, "port": 8188})
prompt = text_prompt()
prompt["collector"]["inputs"]["load_balance"] = True
kwargs = {"enabled_worker_ids": ["pool", "direct"]} if explicit else {}
with pytest.raises(ValueError, match="direct workers"):
run(h, prompt, delegate_master=False, **kwargs)
assert_no_dispatch(h, no_probe=True)
prompt = collector_prompt()
original = copy.deepcopy(prompt)
assert run(h, prompt, delegate_master=False, **kwargs)[2] == 3
assert prompt == original
h.q.dispatch_worker_prompt.assert_awaited_once()
master = h.q.queue_prompt_payload.call_args.args[0]
assert json.loads(master["collector"]["inputs"]["enabled_worker_ids"]) == [
"serverless:pool:1", "serverless:pool:2", "direct"]
jobs = h.manager.start.call_args.args[2]
assert len(jobs) == 2 and all(j["media"]["profile"] == PROFILE for j in jobs)
if direct_type != "local":
h.q.sync_worker_media.assert_awaited_once()
def test_disabled_or_unselected_incompatible_workers_do_not_block(harness):
def test_disabled_or_unselected_workers_do_not_participate(harness):
h = harness
h.config["workers"] += [
{"id": "direct", "type": "local", "enabled": False},
@@ -262,27 +281,32 @@ def test_disabled_or_unselected_incompatible_workers_do_not_block(harness):
assert run(h, enabled_worker_ids=["pool"])[2] == 2
@pytest.mark.parametrize("other", ["DistributedCollector", "UltimateSDUpscaleDistributed", "DistributedImageListStart"])
@pytest.mark.parametrize("workers", [True, False])
def test_mixed_collectors_or_image_list_rejected_even_master_only(harness, other, workers):
h = harness
def test_image_list_cannot_mix_with_collector_even_master_only(harness, workers):
if not workers:
h.config["workers"] = []
prompt = text_prompt()
prompt["incompatible"] = node(other)
harness.config["workers"] = []
prompt = collector_prompt()
prompt["incompatible"] = node("DistributedImageListStart")
with pytest.raises(ValueError, match="cannot be mixed"):
run(h, prompt, delegate_master=False)
assert_no_dispatch(h, no_probe=True)
run(harness, prompt, delegate_master=False)
assert_no_dispatch(harness, no_probe=True)
def test_mixed_pool_profiles_rejected_before_load_balancing(harness):
def test_mixed_direct_typed_and_legacy_profiles_keep_each_wire_contract(harness):
h = harness
h.config["workers"].append(dict(h.pool, id="legacy", media_profile="legacy-image-v1"))
prompt = text_prompt()
prompt["collector"]["inputs"]["load_balance"] = True
with pytest.raises(ValueError, match="requires"):
run(h, prompt, delegate_master=False)
assert_no_dispatch(h, no_probe=True)
h.config["workers"] += [dict(h.pool, id="legacy", media_profile="legacy-image-v1"),
{"id": "direct", "type": "local", "enabled": True}]
assert run(h, collector_prompt())[2] == 5
h.q.dispatch_worker_prompt.assert_awaited_once()
jobs = h.manager.start.call_args.args[2]
assert len(jobs) == 4
for job in jobs:
if job["worker"]["media_profile"] == PROFILE:
assert job["media"]["outputs"] == job["manifest"]
assert h.validator.validate_prepared_job(job) is True
else:
assert "media" not in job
assert any(n["class_type"] == "SaveImage" for n in job["workflow"].values())
def test_typed_master_only_is_supported_without_serverless_work(harness):
@@ -291,27 +315,23 @@ def test_typed_master_only_is_supported_without_serverless_work(harness):
assert run(h, delegate_master=False)[2] == 0
master = h.q.queue_prompt_payload.call_args.args[0]
assert master["collector"]["inputs"]["enabled_worker_ids"] == "[]"
assert master["collector"]["inputs"]["texts"] == ["text", 0]
assert master["collector"]["inputs"]["images"] == ["source", 0]
h.manager.start.assert_not_awaited()
h.q.dispatch_worker_prompt.assert_not_awaited()
@pytest.mark.parametrize("offline", [False, True])
def test_typed_delegate_requires_available_worker_without_master_fallback(harness, offline):
def test_collector_delegate_without_workers_preserves_master_fallback(harness, offline):
h = harness
if offline:
h.q.select_active_workers.side_effect = lambda *a, **kw: ([], False)
else:
h.config["workers"] = []
with pytest.raises(RuntimeError, match="comfy-media-v1"):
run(h)
assert_no_dispatch(h)
def test_unknown_explicit_typed_worker_is_rejected(harness):
with pytest.raises(ValueError, match="Unknown explicitly selected"):
run(harness, enabled_worker_ids=["unknown"], delegate_master=False)
assert_no_dispatch(harness, no_probe=True)
assert run(h)[2] == 0
master = h.q.queue_prompt_payload.call_args.args[0]
assert master["collector"]["inputs"]["delegate_only"] is False
assert master["collector"]["inputs"]["images"] == ["source", 0]
h.manager.start.assert_not_awaited()
@pytest.mark.parametrize("bad", [" pool ", "pool white", "pool\n"])
@@ -323,9 +343,9 @@ def test_pool_media_identity_is_not_repaired(harness, bad):
def test_integer_source_node_identity_is_not_normalized(harness):
prompt = text_prompt()
prompt[1] = prompt.pop("text")
prompt["collector"]["inputs"]["texts"] = [1, 0]
prompt = collector_prompt()
prompt[1] = prompt.pop("source")
prompt["collector"]["inputs"]["images"] = [1, 0]
with pytest.raises(ValueError, match="identifier"):
run(harness, prompt)
assert_no_dispatch(harness, no_probe=True)
@@ -334,18 +354,18 @@ def test_integer_source_node_identity_is_not_normalized(harness):
@pytest.mark.parametrize("broken", ["dangling-unused", "literal", "missing-file", "bad-loader-controls", "oversize"])
def test_compilation_packing_and_size_errors_precede_all_dispatch(harness, media_paths, broken):
h = harness
prompt = text_prompt()
prompt = collector_prompt()
if broken == "dangling-unused":
prompt["unused"] = node("PreviewImage", images=["missing", 0])
elif broken == "literal":
prompt["collector"]["inputs"]["texts"] = "must use a linked string node"
prompt["collector"]["inputs"]["images"] = "must use a linked image node"
elif broken in ("missing-file", "bad-loader-controls"):
prompt["audio"] = node("LoadAudio", audio="missing.wav")
prompt["collector"]["inputs"]["audio"] = ["audio", 0]
if broken == "bad-loader-controls":
prompt["audio"]["inputs"]["duration"] = 5
else:
prompt["text"]["inputs"]["value"] = "雪" * 1000
prompt["source"]["inputs"]["label"] = "雪" * 1000
original = copy.deepcopy(prompt)
with patch.object(h.client, "MAX_REQUEST_BYTES", 100 if broken == "oversize" else h.client.MAX_REQUEST_BYTES):
with pytest.raises((ValueError, OSError)):
@@ -418,9 +438,9 @@ def test_master_queue_failure_cleans_queues_and_never_starts_manager(harness):
def test_manager_setup_failure_emits_one_terminal_per_collector_participant(harness):
h = harness
prompt = text_prompt()
prompt["image"] = node("EmptyImage", width=2, height=2, batch_size=1, color=0)
prompt["collector"]["inputs"]["images"] = ["image", 0]
prompt = collector_prompt()
prompt["audio"] = node("GeneratedAudio")
prompt["collector"]["inputs"]["audio"] = ["audio", 0]
h.manager.start.side_effect = ValueError("journal failure")
with pytest.raises(ValueError, match="journal failure"):
run(h, prompt)
@@ -481,6 +501,7 @@ import comfy.cli_args
comfy.cli_args.args.cpu = True
import torch
def forbidden(*args, **kwargs):
raise AssertionError('CPU audio integration may not initialize a GPU')
torch.cuda.is_available = lambda: False
@@ -507,7 +528,7 @@ raise SystemExit(pytest.main([sys.argv[2], '-q', '-p', 'no:cacheprovider', '-k',
stream.setframerate(8000)
stream.writeframes(pcm.numpy().tobytes())
original_bytes = path.read_bytes()
prompt = text_prompt()
prompt = collector_prompt()
prompt["audio"] = node("LoadAudio", audio="tone.wav")
prompt["collector"]["inputs"]["audio"] = ["audio", 0]
original = copy.deepcopy(prompt)
+37 -17
View File
@@ -6,16 +6,16 @@ import pytest
from test_serverless_media_prompt import compiler, _prompt
from test_media_handler import handler
from test_media_collector import module, collect, native_media
from test_collector_typed_media import collector_module, native_media, assert_exact
FORMATS = ('exact', 'legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1')
FORMATS = ('legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1', 'exact')
def compile_request(fmt='exact', quality=90):
def compile_request(fmt='legacy_png', quality=90):
prompt = _prompt()
prompt['source:1'] = {'class_type': 'Source', 'inputs': {}}
prompt['collector:1']['inputs'].update(transfer_format=fmt, transfer_quality=quality,
masks=['source:1', 1], texts=['source:1', 2])
audio=['source:1', 1])
original = copy.deepcopy(prompt)
workflow, outputs = compiler.compile_media_prompt(prompt, 'request:1')
assert prompt == original
@@ -27,16 +27,23 @@ def compile_request(fmt='exact', quality=90):
@pytest.mark.parametrize('fmt', FORMATS)
def test_compiler_and_handler_preserve_image_policy_only(fmt):
data = compile_request(fmt, 73)
assert {b['socket'] for b in data['media']['outputs']} == {'images', 'audio'}
for binding in data['media']['outputs']:
inputs = data['workflow'][binding['node_id']]['inputs']
if binding['kind'] == 'IMAGE' and fmt != 'exact':
assert inputs['transfer_format'] == fmt
assert inputs['transfer_format'] == ('png' if fmt == 'legacy_png' else fmt)
assert inputs['transfer_quality'] == 73
else:
assert 'transfer_format' not in inputs and 'transfer_quality' not in inputs
handler._validate_request(data)
def test_saved_collector_defaults_compile_to_png_not_exact():
workflow, bindings = compiler.compile_media_prompt(_prompt(), 'request:1')
inputs = workflow[bindings[0]['node_id']]['inputs']
assert (inputs['transfer_format'], inputs['transfer_quality']) == ('png', 90)
@pytest.mark.parametrize('fmt,quality', [('JPEG',90),('',90),('jpeg',True),('jpeg',0),
('jpeg',101),('jpeg','90'),(['source:1',0],90)])
def test_compiler_rejects_invalid_or_linked_controls(fmt, quality):
@@ -54,23 +61,36 @@ def test_handler_rejects_tampered_policy(field, value):
handler._validate_request(data)
def test_handler_rejects_image_controls_on_mask_export():
data = compile_request()
binding = next(b for b in data['media']['outputs'] if b['kind'] == 'MASK')
data['workflow'][binding['node_id']]['inputs']['transfer_format'] = 'jpeg'
@pytest.mark.parametrize('socket,kind', [('audio', 'AUDIO'), ('masks', 'MASK'), ('texts', 'STRING'), ('video', 'VIDEO')])
def test_handler_rejects_image_controls_on_nonimage_bridge_exports(socket, kind):
# The bridge still supports all five kinds; these are direct bridge exports,
# not nonexistent sockets on DistributedCollector.
data = compile_request('exact')
binding = next(b for b in data['media']['outputs'] if b['kind'] == 'AUDIO')
binding.update(socket=socket, kind=kind)
inputs = data['workflow'][binding['node_id']]['inputs']
inputs.pop('audio')
inputs.update({socket: ['source:1', 1], 'binding_json': json.dumps(binding)})
handler._validate_request(data)
inputs['transfer_format'] = 'jpeg'
with pytest.raises(ValueError):
handler._validate_request(data)
@pytest.mark.parametrize('fmt', FORMATS)
def test_master_passthrough_stays_native_for_every_format(module, fmt):
def test_master_passthrough_stays_native_for_every_format(collector_module, fmt):
media = native_media()
actual = collect(module, **media, transfer_format=[fmt], transfer_quality=[73])
for values, original in zip(actual, media.values()):
assert values is original
actual = collector_module.DistributedCollectorNode().run(
images=[media['images']], audio=[media['audio']], pass_through=[True],
transfer_format=[fmt], transfer_quality=[73])
assert actual[0] is media['images']
assert actual[1] is media['audio']
assert_exact(actual[0], media['images'])
@pytest.mark.parametrize('quality', [True, 0, 101, '90', [90,91]])
def test_node_rejects_bad_quality_even_without_remote_participants(module, quality):
with pytest.raises(ValueError):
collect(module, texts=['hello'], transfer_quality=quality)
def test_collector_widget_contract_keeps_saved_defaults_and_appends_exact(collector_module):
optional = collector_module.DistributedCollectorNode.INPUT_TYPES()['optional']
assert list(optional) == ['images', 'audio', 'transfer_format', 'transfer_quality']
assert tuple(optional['transfer_format'][0]) == FORMATS
assert optional['transfer_format'][1]['default'] == 'legacy_png'
assert optional['transfer_quality'][1]['default'] == 90
+23 -18
View File
@@ -24,7 +24,7 @@ compiler = importlib.import_module(f"{PACKAGE}.api.orchestration.serverless_medi
legacy = importlib.import_module(f"{PACKAGE}.api.orchestration.serverless_prompt")
contract = importlib.import_module(f"{PACKAGE}.serverless_bridge.contract")
compile_media_prompt = compiler.compile_media_prompt
SOCKETS = {"images": "IMAGE", "audio": "AUDIO", "masks": "MASK", "texts": "STRING", "video": "VIDEO"}
SOCKETS = {"images": "IMAGE", "audio": "AUDIO"}
def _node(class_type, **inputs):
@@ -33,7 +33,7 @@ def _node(class_type, **inputs):
def _collector(**sockets):
return _node(
"DistributedMediaCollector", multi_job_id="job/42", worker_id="worker_0", **sockets
"DistributedCollector", multi_job_id="job/42", worker_id="worker_0", **sockets
)
@@ -72,26 +72,26 @@ def test_one_export_for_each_typed_socket_with_exact_canonical_binding(socket, k
"request_id": "request:1",
"binding_json": json.dumps(binding),
socket: ["source", 0],
**({"transfer_format": "png", "transfer_quality": 90} if kind == "IMAGE" else {}),
},
}
assert set(graph) == {"source", binding["node_id"]}
def test_mixed_collectors_all_five_kinds_have_independent_dispatch_targets():
def test_multiple_collectors_have_independent_image_audio_targets():
prompt = {
"shared": _node("MultiMediaSource"),
"collector:z": _collector(audio=["shared", 1], video=["shared", 4]),
"collector:a": _collector(images=["shared", 0], masks=["shared", 2], texts=["shared", 3]),
"collector:z": _collector(audio=["shared", 1]),
"collector:a": _collector(images=["shared", 0], audio=["shared", 1]),
}
prompt["collector:z"]["inputs"].update(multi_job_id="job:z", worker_id="participant:z")
graph, bindings = _run(prompt)
assert [(b["collector_id"], b["socket"]) for b in bindings] == [
("collector:a", "images"), ("collector:a", "masks"), ("collector:a", "texts"),
("collector:z", "audio"), ("collector:z", "video"),
("collector:a", "images"), ("collector:a", "audio"), ("collector:z", "audio"),
]
assert len({b["binding_id"] for b in bindings}) == 5
assert len({b["binding_id"] for b in bindings}) == 3
targets = {b["node_id"] for b in bindings}
assert len(targets) == 5
assert len(targets) == 3
assert set(graph) == {"shared"} | targets
for binding in bindings:
export = graph[binding["node_id"]]
@@ -119,12 +119,15 @@ def test_prunes_descendants_and_unrelated_outputs_preserving_required_output_uti
assert not {"load_balance", "pass_through"} & graph[bindings[0]["node_id"]]["inputs"].keys()
def test_stock_primitive_string_preserves_empty_and_link_looking_text():
def test_upstream_primitive_string_preserves_empty_and_link_looking_text():
for text in ("", '["source:1", 0]', "hello\nworld"):
prompt = {"text": _node("PrimitiveString", value=text), "collector": _collector(texts=["text", 0])}
prompt = {"text": _node("PrimitiveString", value=text),
"source": _node("TextToImage", text=["text", 0]),
"collector": _collector(images=["source", 0])}
graph, bindings = _run(prompt)
assert graph["text"] == prompt["text"]
assert graph[bindings[0]["node_id"]]["inputs"]["texts"] == ["text", 0]
assert graph["source"]["inputs"]["text"] == ["text", 0]
assert graph[bindings[0]["node_id"]]["inputs"]["images"] == ["source", 0]
@pytest.mark.parametrize("class_type,inputs", [
@@ -176,11 +179,13 @@ def test_distributed_text_value_can_be_wrapped_by_primitive_string():
prompt = {
"value": _node("DistributedModelName", text="model.safetensors"),
"text": _node("PrimitiveString", value=["value", 0]),
"collector": _collector(texts=["text", 0]),
"source": _node("TextToImage", text=["text", 0]),
"collector": _collector(images=["source", 0]),
}
graph, bindings = _run(prompt)
assert graph["text"]["inputs"]["value"] == "model.safetensors"
assert graph[bindings[0]["node_id"]]["inputs"]["texts"] == ["text", 0]
assert graph["source"]["inputs"]["text"] == ["text", 0]
assert graph[bindings[0]["node_id"]]["inputs"]["images"] == ["source", 0]
assert "value" not in graph
@@ -321,7 +326,7 @@ def test_rejects_invalid_links_everywhere_before_pruning(location, link):
@pytest.mark.parametrize("class_type", [
"DistributedCollector", "DistributedEmptyImage", "UltimateSDUpscaleDistributed",
"DistributedMediaCollector", "DistributedEmptyImage", "UltimateSDUpscaleDistributed",
"DistributedImageListStart", "DistributedImageListEnd", "DistributedWorkerSink",
"ImageBatchDivider", "AudioBatchDivider", "CustomDistributedNode", "custombatchdivider",
])
@@ -353,9 +358,9 @@ def test_rejects_collector_chains_across_different_media(indirect):
prompt = _prompt()
source = "collector:1"
if indirect:
prompt["processing"] = _node("ImageToVideo", images=[source, 0])
prompt["processing"] = _node("ImageToAudio", images=[source, 0])
source = "processing"
prompt["collector:2"] = _collector(video=[source, 0])
prompt["collector:2"] = _collector(audio=[source, 0])
with pytest.raises(ValueError, match="collector-to-collector"):
_run(prompt)
@@ -413,7 +418,7 @@ def guarded_import(name, *args, **kwargs):
return original_import(name, *args, **kwargs)
builtins.__import__ = guarded_import
module = importlib.import_module(name + ".api.orchestration.serverless_media_prompt")
prompt = {"s": {"class_type": "Source", "inputs": {}}, "c": {"class_type": "DistributedMediaCollector", "inputs": {"texts": ["s", 0], "multi_job_id": "job", "worker_id": "worker"}}}
prompt = {"s": {"class_type": "Source", "inputs": {}}, "c": {"class_type": "DistributedCollector", "inputs": {"images": ["s", 0], "multi_job_id": "job", "worker_id": "worker"}}}
graph, bindings = module.compile_media_prompt(prompt, "request")
assert graph[bindings[0]["node_id"]]["class_type"] == "MediaExportV1"
'''
+17
View File
@@ -133,6 +133,7 @@ def test_quality_to_crf_mapping():
def test_normalize_format_rejects_unknown_values():
assert tc.normalize_format(" H264 ") == "h264"
assert tc.normalize_format(" EXACT ") == "exact"
assert tc.normalize_format(None) == tc.DEFAULT_TRANSFER_FORMAT
with pytest.raises(tc.TransferCodecError):
tc.normalize_format("gif")
@@ -158,6 +159,22 @@ def test_decode_chunk_validates_envelope_against_payload():
tc.decode_chunk({**video_env, "count": 5}, video["parts"])
def test_exact_is_appended_without_changing_saved_format_defaults():
assert tc.TRANSFER_FORMATS == (
"legacy_png", "png", "ffv1", "jpeg", "webp", "h264", "h265", "av1", "exact",
)
assert tc.DEFAULT_TRANSFER_FORMAT == "legacy_png"
assert tc.EXACT_FORMAT not in tc.IMAGE_FORMATS + tc.VIDEO_FORMATS
def test_exact_does_not_enter_uint8_chunk_codec():
frames = tc.tensor_batch_to_uint8(_smooth_frames(count=1))
with pytest.raises(tc.TransferCodecError, match="exact"):
list(tc.encode_chunks(frames, "exact", 90, 10**9))
with pytest.raises(tc.TransferCodecError, match="exact"):
tc.decode_chunk({"format": "exact", "count": 1, "width": 91, "height": 67}, [b"raw tensor"])
def test_legacy_format_is_not_chunk_encoded():
frames = tc.tensor_batch_to_uint8(_smooth_frames(count=1))
with pytest.raises(tc.TransferCodecError):
+7 -5
View File
@@ -69,8 +69,8 @@ def v3(monkeypatch):
module(f"{PKG}.nodes.utilities")
module(f"{PKG}.nodes.distributed_upscale", UltimateSDUpscaleDistributed=object)
module(f"{PKG}.nodes.collector", DistributedCollectorNode=CollectorRuntime)
module(f"{PKG}.nodes.image_list", DistributedImageListStart=object, DistributedImageListEnd=object)
module(f"{PKG}.nodes.media_collector", DistributedMediaCollector=object)
module(f"{PKG}.nodes.image_list", DistributedImageListStart=object, DistributedImageListEnd=object,
DistributedImageListGetValue=object)
module("comfy", __path__=[])
module("comfy.samplers", KSampler=types.SimpleNamespace(SAMPLERS=["euler"], SCHEDULERS=["normal"]))
module("comfy_api", __path__=[])
@@ -98,6 +98,7 @@ def test_registered_collector_schema_exposes_optional_transfer_inputs(v3):
fmt = inputs["transfer_format"]
assert fmt.kind == "Combo"
assert fmt.kwargs["options"] == list(codec.TRANSFER_FORMATS)
assert fmt.kwargs["options"][-1] == "exact"
assert fmt.kwargs["default"] == "legacy_png"
assert fmt.kwargs["optional"] is True
quality = inputs["transfer_quality"]
@@ -107,13 +108,14 @@ def test_registered_collector_schema_exposes_optional_transfer_inputs(v3):
assert schema.is_input_list is True
def test_registered_collector_forwards_transfer_options(v3):
@pytest.mark.parametrize("fmt", ["av1", "exact"])
def test_registered_collector_forwards_transfer_options(v3, fmt):
mod, _, calls = v3
mod.DistributedCollector.execute(images=["img"], audio=[None], transfer_format=["av1"], transfer_quality=[70],
mod.DistributedCollector.execute(images=["img"], audio=[None], transfer_format=[fmt], transfer_quality=[70],
multi_job_id=["job"], is_worker=[True])
assert calls[-1]["transfer_format"] == ["av1"]
assert calls[-1]["transfer_format"] == [fmt]
assert calls[-1]["transfer_quality"] == [70]
assert calls[-1]["multi_job_id"] == ["job"]
+1
View File
@@ -34,6 +34,7 @@ def test_image_list_network_runtime():
)
assert result.returncode == 0, result.stdout + '\n' + result.stderr
assert 'IMAGE_LIST_NETWORK_OK' in result.stdout
assert 'IMAGE_LIST_VALUES_OK' in result.stdout
assert 'IMAGE_LIST_CANCEL_OK' in result.stdout
assert 'IMAGE_LIST_MASTER_OK' in result.stdout
assert 'IMAGE_LIST_PREFLIGHT_OK' in result.stdout