feat: integrate native image-list caching and exact serverless value overrides
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
"""Offline CPU proof against ComfyUI's real signatures and PromptExecutor.
|
||||
|
||||
Exercises one native worker executor and separate private master-slot lifetimes,
|
||||
including their shared stable Start / fresh End contract.
|
||||
"""
|
||||
import asyncio
|
||||
import copy
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import types
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def main(core):
|
||||
sys.path.insert(0, str(Path(core).resolve()))
|
||||
import comfy.cli_args
|
||||
comfy.cli_args.args.cpu = True # Must precede all model-management imports.
|
||||
comfy.cli_args.args.disable_assets = True
|
||||
import execution
|
||||
import nodes
|
||||
import torch
|
||||
from comfy_execution.caching import CacheKeySetInputSignature
|
||||
|
||||
name = '_native_cache_identity_probe'
|
||||
for suffix in ('', '.image_list', '.nodes', '.utils'):
|
||||
package = types.ModuleType(name + suffix)
|
||||
package.__path__ = [str(ROOT / suffix.lstrip('.').replace('.', '/'))]
|
||||
sys.modules[name + suffix] = package
|
||||
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('offline native proof attempted a server-loop bridge')
|
||||
|
||||
sys.modules[name + '.utils.async_helpers'] = types.SimpleNamespace(run_async_in_server_loop=forbidden)
|
||||
rt = importlib.import_module(name + '.image_list.runtime')
|
||||
boundary = importlib.import_module(name + '.nodes.image_list')
|
||||
calls = []
|
||||
|
||||
class Operation:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {'required': {'image': ('IMAGE',), 'mask': ('MASK',),
|
||||
'text': ('STRING', {'forceInput': True})}}
|
||||
RETURN_TYPES = ('IMAGE', 'MASK', 'STRING')
|
||||
FUNCTION = 'run'
|
||||
|
||||
def run(self, image, mask, text):
|
||||
calls.append(text)
|
||||
return image + 1, mask + 1, text + ':done'
|
||||
|
||||
nodes.NODE_CLASS_MAPPINGS.update(DistributedImageListStart=boundary.DistributedImageListStart,
|
||||
DistributedImageListEnd=boundary.DistributedImageListEnd,
|
||||
Operation=Operation)
|
||||
prompt = {
|
||||
'start': {'class_type': rt.START_CLASS, 'inputs': {'image_list_execution_id': 'outer-A'}},
|
||||
'op': {'class_type': 'Operation', 'inputs': {
|
||||
'image': ['start', 0], 'mask': ['start', 1], 'text': ['start', 4]}},
|
||||
'end': {'class_type': rt.END_CLASS, 'inputs': {
|
||||
'images': ['op', 0], 'masks': ['op', 1], 'texts': ['op', 2],
|
||||
'context': ['start', 3], 'value_3': ['start', 2]}},
|
||||
}
|
||||
data = rt.normalize_image_list_inputs(torch.zeros(1, 2, 3, 3, dtype=torch.float64),
|
||||
torch.ones(1, 2, 3, dtype=torch.bfloat16),
|
||||
{'value_1': 'original', 'value_2': 3})[0]
|
||||
region = rt.validate_image_list_regions(prompt)[0]
|
||||
|
||||
def prepare(data):
|
||||
token = rt.create_worker_task_token()
|
||||
context = rt.create_master_job('start', 'end', prompt, [data], ['worker'], '')
|
||||
rt.register_assignment(context, data['index'], 'worker', token)
|
||||
identity = rt.store_worker_item(token, rt.encode_item(data))
|
||||
result = rt.build_worker_prompt(prompt, region, token, context, data['index'], '', item_identity=identity)
|
||||
result['start']['inputs']['image_list_local_execution'] = True
|
||||
result['end']['inputs']['image_list_local_execution'] = True
|
||||
return token, context, result
|
||||
|
||||
async def signature(graph, executor):
|
||||
dynamic = execution.DynamicPrompt(graph)
|
||||
changed = execution.IsChangedCache('signature-only', dynamic, executor.caches.outputs)
|
||||
keys = CacheKeySetInputSignature(dynamic, graph.keys(), changed)
|
||||
await keys.add_keys(graph.keys())
|
||||
return {node: keys.get_data_key(node) for node in graph}
|
||||
|
||||
async def probe():
|
||||
server = types.SimpleNamespace(client_id=None, last_node_id=None, send_sync=lambda *a, **kw: None)
|
||||
executor = execution.PromptExecutor(server, cache_type=execution.CacheType.LRU,
|
||||
cache_args={'lru': 64, 'ram': 0, 'ram_inactive': 0})
|
||||
ta, ca, a = prepare(data)
|
||||
prompt['start']['inputs']['image_list_execution_id'] = 'outer-B'
|
||||
tb, cb, b = prepare(data)
|
||||
assert ta != tb and ca != cb
|
||||
ka, kb = await signature(a, executor), await signature(b, executor)
|
||||
assert ka['start'] == kb['start'] and ka['op'] == kb['op']
|
||||
assert ka['end'] != kb['end']
|
||||
assert a['start']['inputs'] == b['start']['inputs']
|
||||
# Exercise actual V3 list mapping, hidden.prompt/unique_id, lazy inputs,
|
||||
# native node signatures and cache hits, not a reimplementation of them.
|
||||
for number, (token, context, graph) in enumerate(((ta, ca, a), (tb, cb, b))):
|
||||
valid = await execution.validate_prompt(f'native-{number}', graph, ['end'])
|
||||
assert valid[0], valid
|
||||
await executor.execute_async(graph, f'native-{number}', {}, ['end'])
|
||||
assert executor.success, executor.status_messages
|
||||
assert token not in rt._WORKER_ITEMS
|
||||
result = rt.get_master_job(context).results[0]
|
||||
assert torch.equal(result['image'], data['image'] + 1)
|
||||
assert torch.equal(result['mask'], data['mask'] + 1)
|
||||
assert result['text'] == 'original:done'
|
||||
assert result['values'] == {'value_3': 0}
|
||||
cached = next(entry['nodes'] for event, entry in executor.status_messages if event == 'execution_cached')
|
||||
if number:
|
||||
assert 'start' in cached and 'op' in cached and 'end' not in cached, cached
|
||||
assert calls == ['original'], calls
|
||||
for failure in ('expired', 'mismatched'):
|
||||
token, context, graph = prepare(data)
|
||||
expires, staged = rt._WORKER_ITEMS[token]
|
||||
if failure == 'expired':
|
||||
rt._WORKER_ITEMS[token] = (0, staged)
|
||||
else:
|
||||
staged['image'] = staged['image'] + 9
|
||||
await executor.execute_async(graph, f'rejected-{failure}', {}, ['end'])
|
||||
assert not executor.success, failure
|
||||
assert not rt.get_master_job(context).received_indexes
|
||||
assert token not in rt._WORKER_ITEMS
|
||||
cached = next(entry['nodes'] for event, entry in executor.status_messages if event == 'execution_cached')
|
||||
assert 'start' in cached and 'op' in cached and 'end' not in cached, cached
|
||||
assert calls == ['original'], calls
|
||||
for part in ('image', 'mask', 'value', 'index'):
|
||||
changed = copy.deepcopy(data)
|
||||
if part in ('image', 'mask'):
|
||||
changed[part].reshape(-1)[0] += 1
|
||||
elif part == 'value':
|
||||
changed['values']['value_2'] = 4 # Even an unconnected slot is content.
|
||||
else:
|
||||
changed['index'] = 1
|
||||
token, context, graph = prepare(changed)
|
||||
keys = await signature(graph, executor)
|
||||
assert keys['start'] != ka['start'] and keys['op'] != ka['op'], part
|
||||
await executor.execute_async(graph, f'changed-{part}', {}, ['end'])
|
||||
assert executor.success, executor.status_messages
|
||||
assert token not in rt._WORKER_ITEMS
|
||||
assert rt.get_master_job(context).received_indexes == {changed['index']}
|
||||
assert len(calls) == 5, calls
|
||||
# Combine the real Start/End identity path with cache transfer between
|
||||
# private master slots, not just independent signature/retention probes.
|
||||
local = importlib.import_module(name + '.image_list.local_executor')
|
||||
before = len(calls)
|
||||
for number in range(2):
|
||||
token, context, graph = prepare(data)
|
||||
slot = local.LocalExecutionSlot()
|
||||
try:
|
||||
await asyncio.wrap_future(slot.submit(graph, f'local-{number}', 'end'))
|
||||
result = rt.get_master_job(context).results[0]
|
||||
assert torch.equal(result['image'], data['image'] + 1)
|
||||
assert result['values'] == {'value_3': 0}
|
||||
assert token not in rt._WORKER_ITEMS
|
||||
finally:
|
||||
await slot.close()
|
||||
rt.remove_master_job(context)
|
||||
assert local._RETAINED_OUTPUTS is not None
|
||||
assert len(calls) == before + 1, calls
|
||||
assert not rt._WORKER_ITEMS
|
||||
assert not torch.cuda.is_initialized()
|
||||
print('NATIVE_CACHE_IDENTITY_OK stable Start/op keys; fresh End key; '
|
||||
'1 operation for 2 same-data jobs; 4 changed-content misses; '
|
||||
'expired/mismatched cached inputs rejected; current delivery and cleanup')
|
||||
|
||||
asyncio.run(probe())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[1])
|
||||
@@ -37,14 +37,14 @@ import aiohttp
|
||||
|
||||
torch.set_num_threads(1)
|
||||
SOURCE_CALLS, BRANCH_CALLS, CAPTURED, MODEL_CALLS = [], [], [], []
|
||||
UTILITY_CALLS, ITEM_EXECUTIONS = [], []
|
||||
UTILITY_CALLS, ITEM_EXECUTIONS, VALUE_CAPTURES = [], [], []
|
||||
ALLOW_MASTER = False
|
||||
COUNT = 7
|
||||
|
||||
|
||||
def source_rows():
|
||||
return [{'image': torch.full((1, 5 + i, 7, 3 if i % 2 == 0 else 4), i / 16, dtype=torch.float32),
|
||||
'reference_image': torch.full((1, 2, 3, 3), (i + 1) / 16),
|
||||
'reference_image': torch.full((2, 2, 3, 3), (i + 1) / 16),
|
||||
'mask': torch.full((1, 5 + i, 7), (i + 1) / 16),
|
||||
'text': f'item-{i}', 'strength': (i + 1) / 16, 'seed': 100 + i, 'index': i}
|
||||
for i in range(COUNT)]
|
||||
@@ -92,7 +92,9 @@ def assert_item_execution(prompt, index):
|
||||
from comfy_execution.utils import get_executing_context
|
||||
context = get_executing_context()
|
||||
assert context is not None
|
||||
assert prompt['2']['inputs'].get('worker_item_token'), 'per-item node ran in outer graph'
|
||||
assert prompt['4']['inputs'].get('worker_result_token'), 'per-item node ran in outer graph'
|
||||
assert (prompt['2']['inputs'].get('worker_item_identity')
|
||||
or prompt['2']['inputs'].get('worker_item_token')), 'per-item source binding missing'
|
||||
running, _ = server.PromptServer.instance.prompt_queue.get_current_queue_volatile()
|
||||
assert len(running) == 1, running
|
||||
entry = running[0]
|
||||
@@ -160,17 +162,32 @@ class TestImageOperation:
|
||||
return image + strength, mask * 0.5, f'{text}:seed-{seed}'
|
||||
|
||||
|
||||
class TestValueOverride:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {'required': {'text': ('STRING', {'forceInput': True}),
|
||||
'index': ('INT', {'forceInput': True})}}
|
||||
RETURN_TYPES = ('STRING', 'INT')
|
||||
FUNCTION = 'run'
|
||||
def run(self, text, index):
|
||||
assert WORKER or ALLOW_MASTER, 'override branch executed on delegate-only master'
|
||||
return (text + ':override', 200 + index)
|
||||
|
||||
|
||||
class TestCapture:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {'required': {'images': ('IMAGE',), 'masks': ('MASK',), 'texts': ('STRING', {'forceInput': True})}}
|
||||
return {'required': {'images': ('IMAGE',), 'masks': ('MASK',), 'texts': ('STRING', {'forceInput': True})},
|
||||
'optional': {'values': ('DISTRIBUTED_IMAGE_LIST_VALUES',),
|
||||
'picked': ('STRING', {'forceInput': True})}}
|
||||
INPUT_IS_LIST = True
|
||||
OUTPUT_NODE = True
|
||||
RETURN_TYPES = ()
|
||||
FUNCTION = 'run'
|
||||
def run(self, images, masks, texts):
|
||||
def run(self, images, masks, texts, values=None, picked=None):
|
||||
assert not WORKER, 'post-End capture leaked into a worker prompt'
|
||||
CAPTURED.append((images, masks, texts))
|
||||
VALUE_CAPTURES.append((values, picked))
|
||||
return ()
|
||||
|
||||
|
||||
@@ -179,11 +196,19 @@ async def setup():
|
||||
ps = server.PromptServer(asyncio.get_running_loop(), assets)
|
||||
assert await nodes.load_custom_node(str(ROOT))
|
||||
module = sys.modules[str(ROOT).replace('.', '_x_')]
|
||||
boundary = importlib.import_module(module.__name__ + '.nodes.image_list')
|
||||
encode_result = boundary.encode_result
|
||||
def checked_encode(result, **options):
|
||||
encoded = encode_result(result, **options)
|
||||
if 'values' in encoded:
|
||||
assert set(encoded['values']) == {f'value_{n}' for n in (2, 3, 4, 5, 7, 8, 9, 10)}
|
||||
return encoded
|
||||
boundary.encode_result = checked_encode
|
||||
nodes.NODE_CLASS_MAPPINGS.update(TestImageSource=TestImageSource,
|
||||
TestImageOperation=TestImageOperation, TestCapture=TestCapture,
|
||||
TestCachedModel=TestCachedModel, TestSharedFlag=TestSharedFlag,
|
||||
TestRequiredOutputUtility=TestRequiredOutputUtility,
|
||||
TestDiscardedPreview=TestDiscardedPreview)
|
||||
TestDiscardedPreview=TestDiscardedPreview, TestValueOverride=TestValueOverride)
|
||||
# Match the CORS-enabled direct-worker launch contract for canvas probes.
|
||||
app = web.Application(middlewares=[server.create_cors_middleware('*')])
|
||||
# Register the actual feature routes, not test replacements.
|
||||
@@ -221,7 +246,10 @@ async def worker_main():
|
||||
accepted = json.loads(response.body)
|
||||
_, pending = ps.prompt_queue.get_current_queue_volatile()
|
||||
entry = next(item for item in pending if item[1] == accepted['prompt_id'])
|
||||
submissions.append({'prompt_id': entry[1], 'requested': data.get('partial_execution_targets'),
|
||||
delivery = next(node['inputs'] for node in entry[2].values()
|
||||
if node['class_type'] == 'DistributedImageListEnd')
|
||||
submissions.append({'prompt_id': entry[1], 'item_index': delivery['worker_item_index'],
|
||||
'requested': data.get('partial_execution_targets'),
|
||||
'queued': entry[4], 'node_ids': sorted(entry[2]),
|
||||
'end_ids': [key for key, node in entry[2].items()
|
||||
if node['class_type'] == 'DistributedImageListEnd']})
|
||||
@@ -271,21 +299,45 @@ def graph(callback, worker_ids):
|
||||
'images': ['1', 0], 'mask': ['1', 2],
|
||||
'values.value_1': ['1', 1], 'values.value_2': ['1', 3],
|
||||
'values.value_3': ['1', 4], 'values.value_4': ['1', 5], 'values.value_5': ['7', 0],
|
||||
'values.value_6': ['1', 2],
|
||||
'image_list_enabled_worker_ids': json.dumps(worker_ids), 'image_list_master_callback': callback}},
|
||||
'3': {'class_type': 'TestImageOperation', 'inputs': {
|
||||
key: ['2', index] for key, index in [('image', 0), ('mask', 1), ('index', 2),
|
||||
('reference_image', 4), ('text', 5), ('strength', 6), ('seed', 7), ('flag', 8)]}
|
||||
| {'image': ['8', 0], 'model': ['6', 0]}},
|
||||
'4': {'class_type': 'DistributedImageListEnd', 'inputs': {
|
||||
'images': ['3', 0], 'masks': ['3', 1], 'texts': ['3', 2], 'context': ['2', 3]}},
|
||||
'5': {'class_type': 'TestCapture', 'inputs': {'images': ['4', 0], 'masks': ['4', 1], 'texts': ['4', 2]}},
|
||||
'images': ['3', 0], 'masks': ['3', 1], 'texts': ['3', 2], 'context': ['2', 3],
|
||||
'value_2': ['10', 0], 'value_3': ['2', 6], 'value_4': ['2', 7], 'value_5': ['2', 8],
|
||||
'value_7': ['3', 1], 'value_8': ['2', 4], 'value_9': ['2', 9], 'value_10': ['10', 1]}},
|
||||
'5': {'class_type': 'TestCapture', 'inputs': {'images': ['4', 0], 'masks': ['4', 1], 'texts': ['4', 2],
|
||||
'values': ['4', 3], 'picked': ['get', 0]}},
|
||||
'6': {'class_type': 'TestCachedModel', 'inputs': {}},
|
||||
'7': {'class_type': 'TestSharedFlag', 'inputs': {}},
|
||||
'8': {'class_type': 'TestRequiredOutputUtility', 'inputs': {'image': ['2', 0], 'index': ['2', 2]}},
|
||||
'9': {'class_type': 'TestDiscardedPreview', 'inputs': {'image': ['8', 0]}},
|
||||
'10': {'class_type': 'TestValueOverride', 'inputs': {'text': ['2', 5], 'index': ['2', 2]}},
|
||||
'get': {'class_type': 'DistributedImageListGetValue', 'inputs': {'values': ['4', 3], 'value_name': 'value_2'}},
|
||||
}
|
||||
|
||||
|
||||
def check_carried_values():
|
||||
values, picked = VALUE_CAPTURES[-1]
|
||||
assert len(values) == len(picked) == COUNT
|
||||
for index, (actual, source) in enumerate(zip(values, source_rows())):
|
||||
assert set(actual) == {f'value_{n}' for n in range(1, 11)}, actual.keys()
|
||||
assert torch.equal(actual['value_1'], source['reference_image'])
|
||||
assert actual['value_2'] == picked[index] == f'item-{index}:override'
|
||||
assert actual['value_3'] == source['strength']
|
||||
assert actual['value_4'] == source['seed']
|
||||
assert actual['value_5'] is True
|
||||
assert torch.equal(actual['value_6'], source['mask'])
|
||||
assert torch.equal(actual['value_7'], source['mask'] * 0.5)
|
||||
assert torch.equal(actual['value_8'], source['reference_image'])
|
||||
assert actual['value_8'].shape[0] == 2, 'value tensor batch was split'
|
||||
assert torch.equal(actual['value_9'], source['mask'])
|
||||
assert actual['value_10'] == 200 + index
|
||||
|
||||
|
||||
async def coordinator():
|
||||
global ALLOW_MASTER
|
||||
ps, assets, module, app = await setup()
|
||||
@@ -403,6 +455,7 @@ async def coordinator():
|
||||
assert executor.success, executor.status_messages
|
||||
assert len(SOURCE_CALLS) == 1 and not BRANCH_CALLS and not UTILITY_CALLS
|
||||
assert len(CAPTURED) == 1
|
||||
check_carried_values()
|
||||
images, masks, texts = CAPTURED[0]
|
||||
assert len(images) == len(masks) == len(texts) == COUNT
|
||||
for i, row in enumerate(source_rows()):
|
||||
@@ -430,15 +483,20 @@ async def coordinator():
|
||||
# left unconnected. Region N+1 must not race the prior worker End.
|
||||
chained = {'1': {'class_type': 'EmptyImage', 'inputs': {
|
||||
'height': 8, 'width': 8, 'batch_size': 2, 'color': 0}}}
|
||||
chained['constant'] = {'class_type': 'TestSharedFlag', 'inputs': {}}
|
||||
previous = '1'
|
||||
value_source = 'constant'
|
||||
for start, end in [('2', '3'), ('4', '5'), ('6', '7')]:
|
||||
chained[start] = {'class_type': 'DistributedImageListStart', 'inputs': {
|
||||
'images': [previous, 0]}}
|
||||
'images': [previous, 0], 'values.value_1': [value_source, 0]}}
|
||||
chained[end] = {'class_type': 'DistributedImageListEnd', 'inputs': {
|
||||
'images': [start, 0], 'context': [start, 3]}}
|
||||
value_source = f'get-{end}'
|
||||
chained[value_source] = {'class_type': 'DistributedImageListGetValue', 'inputs': {
|
||||
'values': [end, 3], 'value_name': 'value_1'}}
|
||||
previous = end
|
||||
chained['8'] = {'class_type': 'TestCapture', 'inputs': {
|
||||
'images': ['7', 0], 'masks': ['7', 1], 'texts': ['7', 2]}}
|
||||
'images': ['7', 0], 'masks': ['7', 1], 'texts': ['7', 2], 'values': ['7', 3]}}
|
||||
saved_chained = copy.deepcopy(chained)
|
||||
with patch.object(config_mod, 'load_config', return_value=config), \
|
||||
patch.object(orchestration, 'load_config', return_value=config), \
|
||||
@@ -451,6 +509,7 @@ async def coordinator():
|
||||
assert len(CAPTURED) == 2 and len(CAPTURED[1][0]) == 2
|
||||
assert all(torch.equal(image, torch.zeros((1, 8, 8, 3))) for image in CAPTURED[1][0])
|
||||
assert CAPTURED[1][1:] == ([None, None], [None, None])
|
||||
assert VALUE_CAPTURES[1][0] == [{'value_1': True}, {'value_1': True}]
|
||||
assert not runtime._MASTER_JOBS
|
||||
print('IMAGE_LIST_SEQUENTIAL_OK three regions; one worker; optional outputs omitted')
|
||||
# Same public queue path, now with master participation enabled. The
|
||||
@@ -466,6 +525,8 @@ async def coordinator():
|
||||
if fmt != 'exact':
|
||||
case_prompt['4']['inputs'].update(transfer_format=fmt, transfer_quality=73)
|
||||
saved_case = copy.deepcopy(case_prompt)
|
||||
# Keep cold-run exact-once assertions independent of other modes.
|
||||
importlib.import_module(module.__name__ + '.image_list.local_executor')._RETAINED_OUTPUTS = None
|
||||
executor = execution.PromptExecutor(ps, cache_args={'ram': 0, 'ram_inactive': 0}, asset_manager=assets)
|
||||
before_sources, before_local, before_models = len(SOURCE_CALLS), len(BRANCH_CALLS), len(MODEL_CALLS)
|
||||
before_utilities, before_events = len(UTILITY_CALLS), len(ITEM_EXECUTIONS)
|
||||
@@ -489,7 +550,7 @@ async def coordinator():
|
||||
assert len({event['prompt_id'] for event in ITEM_EXECUTIONS[before_events:]}) == len(local_calls)
|
||||
assert local_calls, f'{mode}: master did not process any items'
|
||||
assert len(MODEL_CALLS) == before_models + 1, 'local model dependency did not stay cached'
|
||||
remote_calls = []
|
||||
remote_assignments = []
|
||||
async with aiohttp.ClientSession() as session:
|
||||
for worker, before_stats in zip(workers, before_worker_stats):
|
||||
async with session.get(f"http://127.0.0.1:{worker['port']}/test-stats") as response:
|
||||
@@ -499,33 +560,69 @@ async def coordinator():
|
||||
submissions = current['submissions'][len(before_stats['submissions']):]
|
||||
events = current['item_executions'][len(before_stats['item_executions']):]
|
||||
assert Counter(utilities) == Counter(calls), (mode, current)
|
||||
assert len(submissions) == len(calls) == len({event['prompt_id'] for event in events})
|
||||
assigned = [submission['item_index'] for submission in submissions]
|
||||
# A repeated item can hit native processing cache even
|
||||
# when End changes return format. Count fresh deliveries,
|
||||
# not recomputation, for exact-once scheduling evidence.
|
||||
assert Counter(calls) <= Counter(assigned), (mode, worker['id'], calls, assigned)
|
||||
assert len(calls) == len({event['prompt_id'] for event in events}), (mode, events)
|
||||
assert {event['prompt_id'] for event in events} <= {
|
||||
submission['prompt_id'] for submission in submissions}
|
||||
for submission in submissions:
|
||||
assert submission['requested'] == submission['queued'] == ['4'], submission
|
||||
assert '8' in submission['node_ids'] and '9' not in submission['node_ids']
|
||||
assert '5' not in submission['node_ids']
|
||||
remote_calls.extend(calls)
|
||||
remote_assignments.extend(assigned)
|
||||
assert not current['running'] and not current['pending'], current
|
||||
if mode.startswith('mixed'):
|
||||
assert remote_calls, 'mixed execution failed to use direct workers'
|
||||
assert remote_assignments, 'mixed execution failed to use direct workers'
|
||||
else:
|
||||
assert not remote_calls and len(local_calls) == COUNT
|
||||
assert Counter(local_calls + remote_calls) == Counter(range(COUNT)), (local_calls, remote_calls)
|
||||
assert not remote_assignments and len(local_calls) == COUNT
|
||||
assert Counter(local_calls + remote_assignments) == Counter(range(COUNT)), (local_calls, remote_assignments)
|
||||
check_carried_values()
|
||||
images, masks, texts = CAPTURED[-1]
|
||||
assert len(images) == len(masks) == len(texts) == COUNT
|
||||
for i, row in enumerate(source_rows()):
|
||||
expected = row['image'] + row['strength']
|
||||
if fmt == 'png' and i in remote_calls:
|
||||
if fmt == 'png' and i in remote_assignments:
|
||||
expected = expected.clone()
|
||||
expected[..., :3] = (expected[..., :3].clamp(0, 1) * 255).to(torch.uint8).float() / 255
|
||||
assert torch.equal(images[i], expected), (mode, i)
|
||||
assert torch.equal(masks[i], row['mask'] * 0.5), (mode, i)
|
||||
assert texts[i] == f'item-{i}:seed-{100 + i}', (mode, texts)
|
||||
assert not runtime._MASTER_JOBS and not runtime._WORKER_ITEMS
|
||||
participation_counts[mode] = {'master': len(local_calls), 'workers': len(remote_calls)}
|
||||
participation_counts[mode] = {'master': len(local_calls), 'workers': len(remote_assignments)}
|
||||
# Repeat the completed master-only queue with a fresh outer executor.
|
||||
# Every item must be delivered again without recomputing its branch.
|
||||
before_sources, before_local = len(SOURCE_CALLS), len(BRANCH_CALLS)
|
||||
before_models, before_utilities = len(MODEL_CALLS), len(UTILITY_CALLS)
|
||||
before_captures = len(CAPTURED)
|
||||
previous_outputs = CAPTURED[-1]
|
||||
executor = execution.PromptExecutor(ps, cache_args={'ram': 0, 'ram_inactive': 0}, asset_manager=assets)
|
||||
with patch.object(config_mod, 'load_config', return_value=config), \
|
||||
patch.object(orchestration, 'load_config', return_value=config), \
|
||||
patch.object(orchestration, 'queue_prompt_payload', enqueue), \
|
||||
patch.object(config_mod, 'get_worker_timeout_seconds', return_value=15):
|
||||
await asyncio.wait_for(orchestration.orchestrate_distributed_execution(
|
||||
prompt, None, 'test', []), 45)
|
||||
assert executor.success, executor.status_messages
|
||||
assert len(SOURCE_CALLS) == before_sources + 1
|
||||
assert len(BRANCH_CALLS) == before_local
|
||||
assert len(MODEL_CALLS) == before_models
|
||||
assert len(UTILITY_CALLS) == before_utilities
|
||||
assert len(CAPTURED) == before_captures + 1
|
||||
for before, after in zip(previous_outputs[:2], CAPTURED[-1][:2]):
|
||||
assert len(before) == len(after) == COUNT
|
||||
assert all(torch.equal(a, b) for a, b in zip(before, after))
|
||||
assert previous_outputs[2] == CAPTURED[-1][2]
|
||||
check_carried_values()
|
||||
assert not runtime._MASTER_JOBS and not runtime._WORKER_ITEMS
|
||||
assert prompt == saved_prompt
|
||||
print('IMAGE_LIST_CACHE_REQUEUE_OK unchanged items reused; fresh ordered delivery and cleanup')
|
||||
print('IMAGE_LIST_MASTER_OK ' + json.dumps(participation_counts))
|
||||
assert len(outer_executions) == 2 + len(participation_counts) and all(row['captures'] == 1 for row in outer_executions)
|
||||
assert len(outer_executions) == 3 + len(participation_counts) and all(row['captures'] == 1 for row in outer_executions)
|
||||
print('IMAGE_LIST_TRANSFER_HTTP_OK mixed-png and master-only-png; masks/text/alpha/local images exact')
|
||||
print('IMAGE_LIST_VALUES_OK passthrough all six types; sparse override/add; Get; sequential Get-to-Start; direct/mixed/master-only')
|
||||
print('IMAGE_LIST_TARGETS_OK ' + json.dumps({
|
||||
'modes': ['delegate-only', 'mixed', 'master-only'],
|
||||
'outer_targets': outer_executions[0]['targets'], 'worker_targets': ['4'],
|
||||
@@ -539,6 +636,12 @@ async def coordinator():
|
||||
del direct_prompt['8'], direct_prompt['9']
|
||||
direct_prompt['3']['inputs']['image'] = ['2', 0]
|
||||
# Real Comfy user-interrupt flag while a real worker executor is active.
|
||||
rows = source_rows()
|
||||
items = runtime.normalize_image_list_inputs(
|
||||
[row['image'] for row in rows], [row['mask'] for row in rows],
|
||||
{f'value_{n}': [row[key] for row in rows]
|
||||
for n, key in ((1, 'reference_image'), (2, 'text'), (3, 'strength'), (4, 'seed'), (6, 'mask'))}
|
||||
| {'value_5': True})
|
||||
slow = workers[1]
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async def slow_stats():
|
||||
@@ -552,11 +655,11 @@ async def coordinator():
|
||||
await asyncio.sleep(0.02)
|
||||
raise AssertionError('worker state transition timed out')
|
||||
before = await until(lambda state: not state['running'] and not state['pending'])
|
||||
context = runtime.create_master_job('2', '4', direct_prompt, source_rows(), ['slow'],
|
||||
context = runtime.create_master_job('2', '4', direct_prompt, items, ['slow'],
|
||||
f'http://127.0.0.1:{port}')
|
||||
with patch.object(config_mod, 'load_config', return_value=config), \
|
||||
patch.object(config_mod, 'get_worker_timeout_seconds', return_value=15):
|
||||
task = asyncio.create_task(runtime.run_image_list_job(context))
|
||||
task = asyncio.create_task(runtime.run_image_list_job(context, include_values=True))
|
||||
active = await until(lambda state: bool(state['running']))
|
||||
owned_id = active['running'][0]
|
||||
comfy.model_management.interrupt_current_processing(True)
|
||||
@@ -574,14 +677,14 @@ async def coordinator():
|
||||
assert not runtime._MASTER_JOBS
|
||||
print('IMAGE_LIST_CANCEL_OK owned prompt stopped; no further dispatch; context removed')
|
||||
# A worker exception before End must fail, not hang or return a partial list.
|
||||
bad_rows = source_rows()
|
||||
bad_rows[0]['seed'] = -1
|
||||
bad_rows = copy.deepcopy(items)
|
||||
bad_rows[0]['values']['value_4'] = -1
|
||||
context = runtime.create_master_job('2', '4', direct_prompt, bad_rows, ['fast'],
|
||||
f'http://127.0.0.1:{port}')
|
||||
with patch.object(config_mod, 'load_config', return_value=config), \
|
||||
patch.object(config_mod, 'get_worker_timeout_seconds', return_value=15):
|
||||
try:
|
||||
await asyncio.wait_for(runtime.run_image_list_job(context), 5)
|
||||
await asyncio.wait_for(runtime.run_image_list_job(context, include_values=True), 5)
|
||||
raise AssertionError('worker exception was ignored')
|
||||
except RuntimeError as exc:
|
||||
assert 'failed image-list item' in str(exc), str(exc)
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Real CPU PromptExecutor + standalone bridge proof, no provider or service."""
|
||||
import asyncio
|
||||
import importlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import types
|
||||
|
||||
from media_end_to_end_check import bootstrap
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
async def check(base):
|
||||
import torch
|
||||
import execution
|
||||
import folder_paths
|
||||
import nodes
|
||||
import server
|
||||
from app.assets.manager import default_asset_manager
|
||||
from comfy_api.v0_0_2 import io
|
||||
|
||||
torch.set_num_threads(1)
|
||||
assert not torch.cuda.is_initialized()
|
||||
assert execution.comfy.model_management.get_torch_device().type == 'cpu'
|
||||
# Deploy ONLY the bridge folder to a separate import root. Import it before
|
||||
# master code: it cannot rely on the Distributed image_list package.
|
||||
shutil.copytree(ROOT / 'serverless_bridge', base / 'endpoint_bridge')
|
||||
sys.path.insert(0, str(base))
|
||||
previously_loaded = set(sys.modules)
|
||||
bridge = importlib.import_module('endpoint_bridge')
|
||||
handler = importlib.import_module('endpoint_bridge.handler')
|
||||
extension = await bridge.comfy_entrypoint()
|
||||
classes = await extension.get_node_list()
|
||||
assert not any('.image_list' in name for name in set(sys.modules) - previously_loaded)
|
||||
calls = []
|
||||
|
||||
class CalculateOverrides(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(node_id='CalculateOverrides', inputs=[io.Image.Input('image'), io.Mask.Input('mask'),
|
||||
*[io.AnyType.Input(name) for name in ('integer', 'boolean', 'floating', 'text', 'reference', 'reference_mask')]],
|
||||
outputs=[io.Image.Output(), io.Mask.Output(), io.String.Output(), io.Int.Output(),
|
||||
io.Float.Output(), io.Boolean.Output()])
|
||||
|
||||
@classmethod
|
||||
def execute(cls, image, mask, integer, boolean, floating, text, reference, reference_mask):
|
||||
assert type(integer) is int and type(boolean) is bool and type(floating) is float
|
||||
calls.append(integer)
|
||||
return io.NodeOutput(reference + image, reference_mask + mask,
|
||||
text + ':calculated', integer + 7, floating / 2, not boolean)
|
||||
|
||||
class MustNotExecute(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(node_id='MustNotExecute', inputs=[io.Image.Input('images')], outputs=[], is_output_node=True)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, images):
|
||||
raise AssertionError('unrelated output was not pruned')
|
||||
|
||||
for cls in [*classes, CalculateOverrides, MustNotExecute]:
|
||||
cls.GET_NODE_INFO_V1()
|
||||
nodes.NODE_CLASS_MAPPINGS[cls.GET_SCHEMA().node_id] = cls
|
||||
|
||||
package = '_serverless_value_master'
|
||||
for suffix in ('', '.api', '.api.orchestration', '.image_list'):
|
||||
module = types.ModuleType(package + suffix)
|
||||
module.__path__ = [str(ROOT.joinpath(*suffix.lstrip('.').split('.')))] if suffix else [str(ROOT)]
|
||||
sys.modules[module.__name__] = module
|
||||
adapter = importlib.import_module(package + '.image_list.serverless')
|
||||
decode = importlib.import_module(package + '.api.orchestration.serverless_typed_results').collect_typed_results
|
||||
rt = adapter.runtime
|
||||
assets = default_asset_manager()
|
||||
ps = server.PromptServer(asyncio.get_running_loop(), assets)
|
||||
|
||||
def node(kind, **inputs):
|
||||
return {'class_type': kind, 'inputs': inputs}
|
||||
|
||||
image = torch.arange(24, dtype=torch.float64).reshape(1, 2, 3, 4) / 997
|
||||
mask = torch.arange(6, dtype=torch.bfloat16).reshape(1, 2, 3) / 16
|
||||
values = {'value_1': 2**63 + 123, 'value_2': False, 'value_3': -0.0, 'value_4': ' 雪\r\n',
|
||||
'value_5': image.repeat(2, 1, 1, 1), 'value_6': mask.repeat(2, 1, 1),
|
||||
'value_7': 'master-only unchanged'}
|
||||
item = {'image': image, 'mask': mask, 'values': values, 'index': 0}
|
||||
graph = {
|
||||
'start': node(rt.START_CLASS),
|
||||
'calculate': node('CalculateOverrides', image=['start', 0], mask=['start', 1],
|
||||
integer=['start', 4], boolean=['start', 5], floating=['start', 6], text=['start', 7],
|
||||
reference=['start', 8], reference_mask=['start', 9]),
|
||||
'end': node(rt.END_CLASS, context=['start', 3], images=['start', 0]),
|
||||
'preview': node('MustNotExecute', images=['start', 0]),
|
||||
}
|
||||
names = ('value_1', 'value_2', 'value_4', 'value_6', 'value_8', 'value_10')
|
||||
expected = dict(zip(names, (values['value_5'] + image, values['value_6'] + mask,
|
||||
values['value_4'] + ':calculated', values['value_1'] + 7, -0.0, True)))
|
||||
graph['end']['inputs'].update({name: ['calculate', index] for index, name in enumerate(names)})
|
||||
region = {'start_id': 'start', 'end_id': 'end', 'node_ids': set(graph)}
|
||||
worker = {'id': 'pool', 'type': 'serverless', 'endpoint_id': 'ep', 'timeout_seconds': 30,
|
||||
'jobs_per_run': 16, 'enabled': True, 'media_profile': 'comfy-media-v1'}
|
||||
|
||||
async def run(prompt, request_id):
|
||||
job = await adapter.prepare_serverless_item(prompt, region, item, worker,
|
||||
request_id=request_id, job_id='job')
|
||||
assert not any('Distributed' in n['class_type'] for n in job['workflow'].values())
|
||||
workflow, media, legacy, targets = handler._validate_request(
|
||||
{key: job[key] for key in ('workflow', 'images', 'media')})
|
||||
stage = handler._Staging(folder_paths.get_input_directory(), request_id)
|
||||
|
||||
async def submit(native):
|
||||
assert 'preview' not in native
|
||||
valid = await execution.validate_prompt(request_id, native, targets)
|
||||
assert valid[0], valid
|
||||
executor = execution.PromptExecutor(ps, cache_args={'ram': 0, 'ram_inactive': 0}, asset_manager=assets)
|
||||
await asyncio.to_thread(executor.execute, native, request_id, {}, targets)
|
||||
assert executor.success, executor.status_messages
|
||||
return executor.history_result
|
||||
|
||||
async def fetch(artifact):
|
||||
transport = artifact['transport']
|
||||
path = Path(folder_paths.get_temp_directory()) / transport['subfolder'] / transport['filename']
|
||||
return path.read_bytes()
|
||||
|
||||
execution_adapter = types.SimpleNamespace(media=media, targets=targets, stage=stage,
|
||||
remaining=lambda: 30, submit_and_wait=submit, fetch=fetch)
|
||||
try:
|
||||
response = await handler._run(execution_adapter, workflow, legacy, None, None)
|
||||
grouped = await decode(response, media, job['manifest'])
|
||||
finally:
|
||||
stage.close(cleanup=True)
|
||||
result = rt.normalize_worker_result(grouped['job']['images'], values={
|
||||
name: grouped['job'][name] for name in rt.VALUE_NAMES if name in grouped['job']})
|
||||
state = rt.ImageListJob('ctx', 'start', 'end', prompt, [item], ['pool'], '')
|
||||
return job, rt._collect_output_lists([result], state=state, include_values=True)[3][0]
|
||||
|
||||
job, actual = await run(graph, 'calculated')
|
||||
assert set(b['socket'] for b in job['manifest']) == {'images', *names}
|
||||
assert calls == [values['value_1']]
|
||||
for name, wanted in {**values, **expected}.items():
|
||||
got = actual[name]
|
||||
assert type(got) is type(wanted), name
|
||||
if isinstance(wanted, torch.Tensor):
|
||||
assert got.dtype == wanted.dtype and got.shape == wanted.shape
|
||||
assert torch.equal(got.contiguous().view(torch.uint8), wanted.contiguous().view(torch.uint8)), name
|
||||
elif isinstance(wanted, float):
|
||||
assert struct.pack('!d', got) == struct.pack('!d', wanted), name
|
||||
else:
|
||||
assert got == wanted, name
|
||||
# Direct overrides exercise exact VALUE import injection for bool/int/float/
|
||||
# string, native tensor import slots, and no execution of the calculation.
|
||||
for name in names:
|
||||
graph['end']['inputs'].pop(name)
|
||||
graph['end']['inputs'].update({f'value_{i + 1}': ['start', i + 4] for i in range(6)})
|
||||
_, direct = await run(graph, 'passthrough')
|
||||
assert calls == [values['value_1']]
|
||||
assert type(direct['value_1']) is int and direct['value_1'] == values['value_1']
|
||||
assert direct['value_2'] is False
|
||||
assert struct.pack('!d', direct['value_3']) == struct.pack('!d', -0.0)
|
||||
assert direct['value_4'] == values['value_4']
|
||||
assert torch.equal(direct['value_5'], values['value_5'])
|
||||
assert torch.equal(direct['value_6'], values['value_6'])
|
||||
assert not torch.cuda.is_initialized()
|
||||
print('IMAGE_LIST_SERVERLESS_VALUES_CPU_OK calculated six types; sparse master overlay; standalone bridge; real PromptExecutor')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(check(bootstrap()))
|
||||
@@ -0,0 +1,114 @@
|
||||
"""CPU-only native Comfy execution across private-slot lifetimes."""
|
||||
import asyncio
|
||||
import importlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import types
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
import comfy.cli_args
|
||||
comfy.cli_args.args.cpu = True
|
||||
comfy.cli_args.args.disable_assets = True
|
||||
import execution
|
||||
import nodes
|
||||
|
||||
pkg = '_local_retention_probe'
|
||||
for name, path in [(pkg, ROOT), (pkg + '.image_list', ROOT / 'image_list')]:
|
||||
module = types.ModuleType(name)
|
||||
module.__path__ = [str(path)]
|
||||
sys.modules[name] = module
|
||||
local = importlib.import_module(pkg + '.image_list.local_executor')
|
||||
|
||||
CALLS = []
|
||||
OUTPUTS = []
|
||||
|
||||
class Counted:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {'required': {'value': ('INT',)}}
|
||||
RETURN_TYPES = ('INT',)
|
||||
FUNCTION = 'run'
|
||||
def run(self, value):
|
||||
if value < 0:
|
||||
raise ValueError('intentional retention failure')
|
||||
CALLS.append(value)
|
||||
return (value,)
|
||||
|
||||
class Uncached(Counted):
|
||||
@classmethod
|
||||
def IS_CHANGED(cls, **kwargs):
|
||||
return float('nan')
|
||||
|
||||
class Capture:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {'required': {'value': ('INT',), 'delivery': ('STRING',)}}
|
||||
RETURN_TYPES = ()
|
||||
OUTPUT_NODE = True
|
||||
FUNCTION = 'run'
|
||||
def run(self, value, delivery):
|
||||
OUTPUTS.append((value, delivery))
|
||||
return ()
|
||||
|
||||
nodes.NODE_CLASS_MAPPINGS.update(RetentionCounted=Counted, RetentionUncached=Uncached, RetentionCapture=Capture)
|
||||
|
||||
async def run(slot, value, delivery, uncached=False):
|
||||
prompt = {
|
||||
'source': {'class_type': 'RetentionUncached' if uncached else 'RetentionCounted', 'inputs': {'value': value}},
|
||||
'end': {'class_type': 'RetentionCapture', 'inputs': {'value': ['source', 0], 'delivery': delivery}},
|
||||
}
|
||||
await asyncio.wrap_future(slot.submit(prompt, delivery, 'end'))
|
||||
|
||||
async def main():
|
||||
first = local.LocalExecutionSlot()
|
||||
await run(first, 1, 'first-1')
|
||||
await run(first, 2, 'first-2')
|
||||
await first.close()
|
||||
assert local._RETAINED_OUTPUTS is not None
|
||||
second = local.LocalExecutionSlot()
|
||||
await run(second, 1, 'second-1')
|
||||
await run(second, 2, 'second-2')
|
||||
await run(second, 3, 'changed')
|
||||
await second.close()
|
||||
assert CALLS == [1, 2, 3], CALLS
|
||||
assert OUTPUTS == [(1, 'first-1'), (2, 'first-2'), (1, 'second-1'), (2, 'second-2'), (3, 'changed')], OUTPUTS
|
||||
third = local.LocalExecutionSlot()
|
||||
await run(third, 4, 'uncached-1', True)
|
||||
await run(third, 4, 'uncached-2', True)
|
||||
await third.close()
|
||||
assert CALLS[-2:] == [4, 4]
|
||||
comfy.cli_args.args.cache_none = True
|
||||
disabled = local.LocalExecutionSlot()
|
||||
await run(disabled, 1, 'disabled-1')
|
||||
await run(disabled, 1, 'disabled-2')
|
||||
await disabled.close()
|
||||
assert CALLS[-2:] == [1, 1]
|
||||
assert local._RETAINED_OUTPUTS is None
|
||||
comfy.cli_args.args.cache_none = False
|
||||
failed = local.LocalExecutionSlot()
|
||||
await run(failed, 7, 'before-failure')
|
||||
try:
|
||||
await run(failed, -1, 'failure')
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('failure fixture unexpectedly succeeded')
|
||||
await failed.close()
|
||||
assert local._RETAINED_OUTPUTS is None
|
||||
cancelled = local.LocalExecutionSlot()
|
||||
await run(cancelled, 8, 'cancelled')
|
||||
cancelled.request_stop()
|
||||
await cancelled.close()
|
||||
assert local._RETAINED_OUTPUTS is None
|
||||
oversized = execution.LRUCache(execution.CacheKeySetInputSignature, max_size=local.NATIVE_CACHE_MAX_ENTRIES)
|
||||
empty = execution.DynamicPrompt({})
|
||||
await oversized.set_prompt(empty, [], execution.IsChangedCache('bound', empty, oversized))
|
||||
oversized.min_generation = oversized.generation
|
||||
oversized.cache = {n: n for n in range(local.NATIVE_CACHE_MAX_ENTRIES + 1)}
|
||||
local._retain_outputs(oversized)
|
||||
assert local._RETAINED_OUTPUTS is None
|
||||
print('PASS: cross-run native cache reuse, fresh delivery, input change, noncacheable nodes, cache_none, cancellation, retention bound')
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -93,11 +93,12 @@ async def check(base):
|
||||
contract = load("serverless_bridge.contract")
|
||||
video_codec = load("serverless_bridge.video_codec")
|
||||
bridge = load("serverless_bridge")
|
||||
collector = load("nodes.media_collector").DistributedMediaCollector
|
||||
collector = load("nodes.v3").DistributedCollector
|
||||
extension = await bridge.comfy_entrypoint()
|
||||
bridge_classes = await extension.get_node_list()
|
||||
assert {cls.GET_SCHEMA().node_id for cls in bridge_classes} == {"MediaImportV1", "MediaExportV1"}
|
||||
sockets = list(contract.SOCKET_KINDS)
|
||||
collector_sockets = ["images", "audio"]
|
||||
kinds = list(contract.SOCKET_KINDS.values())
|
||||
assert kinds == ["IMAGE", "AUDIO", "MASK", "STRING", "VIDEO"]
|
||||
io_types = [io.Image, io.Audio, io.Mask, io.String, io.Video]
|
||||
@@ -165,9 +166,10 @@ async def check(base):
|
||||
assert set(media) == set(expected)
|
||||
for socket in expected:
|
||||
assert len(media[socket]) == len(expected[socket]), (tag, socket)
|
||||
# Collector must preserve native objects and ordered item lists,
|
||||
# not merge batches, resample, convert types, or encode again.
|
||||
assert all(a is e for a, e in zip(media[socket], expected[socket])), (tag, socket)
|
||||
# Existing Collector outputs are a batched IMAGE and one AUDIO,
|
||||
# not lists. Native V3 wraps each output once for this list sink.
|
||||
for actual, wanted in zip(media[socket], expected[socket]):
|
||||
exact(contract.SOCKET_KINDS[socket], actual, wanted)
|
||||
counts = {socket: len(media[socket]) for socket in expected}
|
||||
seen_sinks[tag[0]] = counts
|
||||
return io.NodeOutput(ui={"accepted": [counts]})
|
||||
@@ -180,6 +182,19 @@ async def check(base):
|
||||
def node(class_type, **inputs):
|
||||
return {"class_type": class_type, "inputs": inputs}
|
||||
|
||||
def add_bridge_exports(workflow, manifest, request_id, participant, job_id, links):
|
||||
"""Exercise bridge-only kinds without inventing Collector sockets."""
|
||||
for socket, link in links.items():
|
||||
export_id = "bridge-export:" + socket
|
||||
binding = {"binding_id": "bridge-binding:" + socket, "collector_id": "bridge-only",
|
||||
"job_id": job_id, "participant_id": participant, "node_id": export_id,
|
||||
"socket": socket, "kind": contract.SOCKET_KINDS[socket]}
|
||||
assert export_id not in workflow
|
||||
manifest.append(binding)
|
||||
workflow[export_id] = node("MediaExportV1", request_id=request_id,
|
||||
binding_json=json.dumps(binding), **{socket: link})
|
||||
contract.validate_bindings(manifest)
|
||||
|
||||
def tiny_video(variant):
|
||||
buffer = bytes_io.BytesIO()
|
||||
with av.open(buffer, "w", format="matroska") as container:
|
||||
@@ -199,7 +214,7 @@ async def check(base):
|
||||
image[0, 0, 0, 0] = -0.0
|
||||
mask = torch.tensor([[[-0.0, 0.125, 0.875]]], dtype=torch.bfloat16)
|
||||
audio = {"waveform": torch.arange(38, dtype=torch.float64).reshape(1, 2, 19) / 997,
|
||||
"sample_rate": 44100 + variant}
|
||||
"sample_rate": 44100}
|
||||
audio["waveform"][0, 0, 0] = -0.0
|
||||
component = VideoFromComponents(VideoComponents(
|
||||
images=image[..., :3], frame_rate=Fraction(30000, 1001), alpha=mask.reshape(1, 1, 3, 1).repeat(2, 2, 1, 1),
|
||||
@@ -220,7 +235,7 @@ async def check(base):
|
||||
with wave.open(str(wav), "wb") as out:
|
||||
out.setnchannels(2)
|
||||
out.setsampwidth(2)
|
||||
out.setframerate(22050)
|
||||
out.setframerate(44100)
|
||||
out.writeframes(pcm.tobytes())
|
||||
waveform, sample_rate = nodes_audio.load(str(wav))
|
||||
clip = native_root / "native.mkv"
|
||||
@@ -278,17 +293,22 @@ async def check(base):
|
||||
participant = "serverless:" + key
|
||||
request_id = "acceptance-" + key
|
||||
graph = {"source": node("AcceptanceMediaSource", key=key),
|
||||
"collector": node("DistributedMediaCollector", load_balance=False,
|
||||
"collector": node("DistributedCollector", load_balance=False, transfer_format="exact",
|
||||
multi_job_id="job-main", worker_id=participant,
|
||||
**{socket: ["source", index] for index, socket in enumerate(sockets)}),
|
||||
"text-collector": node("DistributedMediaCollector", load_balance=False,
|
||||
multi_job_id="job-text", worker_id=participant, texts=["source", 3])}
|
||||
**{socket: ["source", index] for index, socket in enumerate(collector_sockets)}),
|
||||
"audio-collector": node("DistributedCollector", load_balance=False,
|
||||
multi_job_id="job-audio", worker_id=participant, audio=["source", 1])}
|
||||
if key == "worker-a":
|
||||
graph.update({"wav": node("LoadAudio", audio=wav.name), "clip": node("LoadVideo", file=clip.name)})
|
||||
graph["collector"]["inputs"].update(audio=["wav", 0], video=["clip", 0])
|
||||
graph["collector"]["inputs"]["audio"] = ["wav", 0]
|
||||
before = copy.deepcopy(graph)
|
||||
workflow, manifest = compiler.compile_media_prompt(graph, request_id)
|
||||
assert graph == before
|
||||
bridge_links = {socket: ["source", sockets.index(socket)] for socket in sockets[2:]}
|
||||
if key == "worker-a":
|
||||
workflow["clip"] = copy.deepcopy(graph["clip"])
|
||||
bridge_links["video"] = ["clip", 0]
|
||||
add_bridge_exports(workflow, manifest, request_id, participant, "job-bridge", bridge_links)
|
||||
assert len(manifest) == 6 and {b["kind"] for b in manifest} == set(kinds)
|
||||
folder_paths.set_input_directory(str(native_root))
|
||||
workflow_before = copy.deepcopy(workflow)
|
||||
@@ -335,17 +355,19 @@ async def check(base):
|
||||
# Exercise the wire format, not an in-memory handler-only object.
|
||||
result = json.loads(json.dumps(result, allow_nan=False))
|
||||
grouped = await decoder.collect_typed_results(result, media, manifest)
|
||||
assert set(grouped) == {"job-main", "job-text"}
|
||||
assert set(grouped["job-main"]) == set(sockets)
|
||||
assert grouped["job-text"]["texts"] == expectations[key]["texts"]
|
||||
assert set(grouped) == {"job-main", "job-audio", "job-bridge"}
|
||||
assert set(grouped["job-main"]) == set(collector_sockets)
|
||||
assert set(grouped["job-bridge"]) == set(sockets[2:])
|
||||
exact("AUDIO", grouped["job-audio"]["audio"][0], fixtures[key]["audio"][0])
|
||||
all_media = {**grouped["job-main"], **grouped["job-bridge"]}
|
||||
for socket, kind in zip(sockets, kinds):
|
||||
values, expected = grouped["job-main"][socket], expectations[key][socket]
|
||||
values, expected = all_media[socket], expectations[key][socket]
|
||||
assert len(values) == len(expected), (key, socket)
|
||||
for actual, wanted in zip(values, expected):
|
||||
exact(kind, actual, wanted)
|
||||
if key == "worker-b":
|
||||
original = fixtures[key]["video"][0].get_components()
|
||||
decoded = grouped["job-main"]["video"][0].get_components()
|
||||
decoded = grouped["job-bridge"]["video"][0].get_components()
|
||||
for field in ("images", "alpha"):
|
||||
assert getattr(decoded, field).data_ptr() != getattr(original, field).data_ptr()
|
||||
assert decoded.audio["waveform"].data_ptr() != original.audio["waveform"].data_ptr()
|
||||
@@ -363,7 +385,7 @@ async def check(base):
|
||||
assert ps.distributed_pending_jobs == {}
|
||||
# Exercise user-selected image return encoding through the actual
|
||||
# compiler, HTTP handler, V3 exporter and atomic response decoder.
|
||||
for fmt in ("png", "jpeg", "h264"):
|
||||
for fmt in ("legacy_png", "png", "jpeg", "h264"):
|
||||
key = "encoded-" + fmt
|
||||
fixtures[key] = native_fixtures(3)
|
||||
# Keep full alpha precision and use nontrivial image dimensions.
|
||||
@@ -372,12 +394,23 @@ async def check(base):
|
||||
fixtures[key]["images"] = [rgba]
|
||||
request_id = "request-" + key
|
||||
graph = {"source": node("AcceptanceMediaSource", key=key),
|
||||
"collector": node("DistributedMediaCollector", transfer_format=fmt,
|
||||
"collector": node("DistributedCollector", transfer_format=fmt,
|
||||
transfer_quality=73, multi_job_id="job-" + key,
|
||||
worker_id="serverless:" + key,
|
||||
**{socket: ["source", i] for i, socket in enumerate(sockets)})}
|
||||
**{socket: ["source", i] for i, socket in enumerate(collector_sockets)})}
|
||||
if fmt == "legacy_png":
|
||||
# A saved Collector with no newly selected controls still uses
|
||||
# legacy_png / 90, mapped to PNG by the serverless compiler.
|
||||
graph["collector"]["inputs"].pop("transfer_format")
|
||||
graph["collector"]["inputs"].pop("transfer_quality")
|
||||
original = copy.deepcopy(graph)
|
||||
workflow, manifest = compiler.compile_media_prompt(graph, request_id)
|
||||
if fmt == "legacy_png":
|
||||
binding = next(b for b in manifest if b["kind"] == "IMAGE")
|
||||
controls = workflow[binding["node_id"]]["inputs"]
|
||||
assert (controls["transfer_format"], controls["transfer_quality"]) == ("png", 90)
|
||||
add_bridge_exports(workflow, manifest, request_id, "serverless:" + key, "job-bridge-" + key,
|
||||
{socket: ["source", sockets.index(socket)] for socket in sockets[2:]})
|
||||
media = {"version": 1, "profile": contract.PROFILE, "request_id": request_id,
|
||||
"artifacts": [], "inputs": [], "outputs": manifest}
|
||||
body = json.loads(serializer(workflow, [], 30, media=media))
|
||||
@@ -385,23 +418,32 @@ async def check(base):
|
||||
input_dir=base / "input", temp_dir=base / "temp", timeout_seconds=30)
|
||||
assert graph == original
|
||||
grouped = await decoder.collect_typed_results(result, media, manifest)
|
||||
actual = grouped["job-" + key]
|
||||
actual = {**grouped["job-" + key], **grouped["job-bridge-" + key]}
|
||||
image = actual["images"][0]
|
||||
assert image.shape == rgba.shape and image.dtype == rgba.dtype
|
||||
assert torch.equal(image[..., 3], rgba[..., 3]), "alpha must not be discarded/narrowed"
|
||||
assert not torch.equal(image[..., :3], rgba[..., :3]), "selection must actually encode"
|
||||
if fmt == "png":
|
||||
if fmt in ("legacy_png", "png"):
|
||||
pixels = (rgba[..., :3].float().clamp(0, 1) * 255).to(torch.uint8)
|
||||
assert torch.equal(image[..., :3], (pixels.float() / 255).to(rgba.dtype))
|
||||
for socket, kind in zip(sockets[1:], kinds[1:]):
|
||||
for got, wanted in zip(actual[socket], fixtures[key][socket]):
|
||||
exact(kind, got, video_codec.encode_video(wanted) if kind == "VIDEO" else wanted)
|
||||
assert not list((base / "temp").rglob("*.bin"))
|
||||
print("IMAGE_TRANSFER_HTTP_OK png jpeg h264; alpha and other media exact")
|
||||
print("REAL_HTTP_BRIDGE_RESULTS_OK all five kinds; 6 bindings/request; native VIDEO remains readable after cleanup")
|
||||
print("IMAGE_TRANSFER_HTTP_OK default legacy_png -> png/90; png jpeg h264; alpha and other media exact")
|
||||
print("REAL_HTTP_BRIDGE_RESULTS_OK Collector IMAGE/AUDIO + independent MASK/STRING/VIDEO bridge exports; native VIDEO survives cleanup")
|
||||
print("ATOMIC_DECODE_REJECTION_OK corrupt final artifact rejected before delivery")
|
||||
|
||||
participants = list(grouped_results)
|
||||
|
||||
def expected_batches(ordered, include_images=True):
|
||||
audio_items = [item for media in ordered for item in media["audio"]]
|
||||
expected = {"audio": [{"waveform": torch.cat([item["waveform"] for item in audio_items], dim=-1),
|
||||
"sample_rate": audio_items[0]["sample_rate"]}]}
|
||||
if include_images:
|
||||
expected["images"] = [torch.cat([image for media in ordered for image in media["images"]])]
|
||||
return expected
|
||||
|
||||
for delegate in (False, True):
|
||||
tag = "delegate" if delegate else "master"
|
||||
job = "collect-" + tag
|
||||
@@ -410,39 +452,40 @@ async def check(base):
|
||||
await queue.put({"worker_id": participant, "media": grouped_results[participant]["job-main"], "is_last": True})
|
||||
ps.distributed_pending_jobs[job] = queue
|
||||
ordered = ([] if delegate else [fixtures["local"]]) + [grouped_results[p]["job-main"] for p in participants]
|
||||
expected_sinks[tag] = {socket: [item for values in ordered for item in values[socket]] for socket in sockets}
|
||||
graph = {"collector": node("DistributedMediaCollector", load_balance=False, multi_job_id=job,
|
||||
enabled_worker_ids=json.dumps(participants), expected_media=json.dumps(sockets), delegate_only=delegate),
|
||||
expected_sinks[tag] = expected_batches(ordered)
|
||||
graph = {"collector": node("DistributedCollector", load_balance=False, multi_job_id=job,
|
||||
enabled_worker_ids=json.dumps(participants), delegate_only=delegate, transfer_format="exact"),
|
||||
"assert": node("AcceptanceMediaAssert", tag=tag,
|
||||
**{socket: ["collector", index] for index, socket in enumerate(sockets)})}
|
||||
**{socket: ["collector", index] for index, socket in enumerate(collector_sockets)})}
|
||||
if not delegate:
|
||||
graph["source"] = node("AcceptanceMediaSource", key="local")
|
||||
graph["collector"]["inputs"].update({socket: ["source", index] for index, socket in enumerate(sockets)})
|
||||
# The second group from the very same complete response must reach its
|
||||
# own Collector, rather than being folded into the first collector job.
|
||||
text_job, text_tag = job + "-text", tag + "-text"
|
||||
text_queue = asyncio.Queue()
|
||||
graph["collector"]["inputs"].update({socket: ["source", index] for index, socket in enumerate(collector_sockets)})
|
||||
# A second audio-only Collector consumes its own job group from the same
|
||||
# atomic response. It must return None for images and batch audio in order.
|
||||
audio_job, audio_tag = job + "-audio", tag + "-audio"
|
||||
audio_queue = asyncio.Queue()
|
||||
for participant in reversed(participants):
|
||||
await text_queue.put({"worker_id": participant,
|
||||
"media": grouped_results[participant]["job-text"], "is_last": True})
|
||||
ps.distributed_pending_jobs[text_job] = text_queue
|
||||
text_ordered = ([] if delegate else [fixtures["local"]]) + [grouped_results[p]["job-text"] for p in participants]
|
||||
expected_sinks[text_tag] = {"texts": [item for values in text_ordered for item in values["texts"]]}
|
||||
graph["text-collector"] = node("DistributedMediaCollector", load_balance=False, multi_job_id=text_job,
|
||||
enabled_worker_ids=json.dumps(participants), expected_media='["texts"]', delegate_only=delegate)
|
||||
graph["text-assert"] = node("AcceptanceMediaAssert", tag=text_tag, texts=["text-collector", 3])
|
||||
await audio_queue.put({"worker_id": participant,
|
||||
"media": grouped_results[participant]["job-audio"], "is_last": True})
|
||||
ps.distributed_pending_jobs[audio_job] = audio_queue
|
||||
audio_ordered = ([] if delegate else [fixtures["local"]]) + [grouped_results[p]["job-audio"] for p in participants]
|
||||
expected_sinks[audio_tag] = expected_batches(audio_ordered, include_images=False)
|
||||
graph["audio-collector"] = node("DistributedCollector", load_balance=False, multi_job_id=audio_job,
|
||||
enabled_worker_ids=json.dumps(participants), delegate_only=delegate)
|
||||
graph["audio-assert"] = node("AcceptanceMediaAssert", tag=audio_tag, audio=["audio-collector", 1])
|
||||
if not delegate:
|
||||
graph["text-collector"]["inputs"]["texts"] = ["source", 3]
|
||||
graph["audio-collector"]["inputs"]["audio"] = ["source", 1]
|
||||
valid = await execution.validate_prompt(job, graph, None)
|
||||
assert valid[0] and not valid[3], valid
|
||||
executor = execution.PromptExecutor(ps, cache_args={"ram": 0, "ram_inactive": 0}, asset_manager=asset_manager)
|
||||
await asyncio.to_thread(executor.execute, graph, job, {}, valid[2])
|
||||
assert executor.success, executor.status_messages
|
||||
assert executor.history_result["outputs"]["assert"]["accepted"] == [seen_sinks[tag]]
|
||||
assert executor.history_result["outputs"]["text-assert"]["accepted"] == [seen_sinks[text_tag]]
|
||||
assert job not in ps.distributed_pending_jobs and text_job not in ps.distributed_pending_jobs
|
||||
assert source_calls == ["worker-a", "worker-b", "encoded-png", "encoded-jpeg", "encoded-h264", "local"], source_calls
|
||||
assert set(seen_sinks) == {"master", "delegate", "master-text", "delegate-text"}
|
||||
assert executor.history_result["outputs"]["audio-assert"]["accepted"] == [seen_sinks[audio_tag]]
|
||||
assert job not in ps.distributed_pending_jobs and audio_job not in ps.distributed_pending_jobs
|
||||
assert source_calls == ["worker-a", "worker-b", "encoded-legacy_png", "encoded-png", "encoded-jpeg", "encoded-h264", "local"], source_calls
|
||||
assert set(seen_sinks) == {"master", "delegate", "master-audio", "delegate-audio"}
|
||||
|
||||
assert not list((base / "output").rglob("*"))
|
||||
assert not torch.cuda.is_initialized()
|
||||
print("MASTER_AND_DELEGATE_ORDER_OK", json.dumps(seen_sinks, sort_keys=True))
|
||||
|
||||
@@ -36,7 +36,7 @@ BASELINE = json.loads((ROOT / 'tests/fixtures/v1_node_contracts.json').read_text
|
||||
ADDED_OPTIONAL_INPUTS = {
|
||||
'DistributedCollector': {
|
||||
'transfer_format': [
|
||||
['legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1'],
|
||||
['legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1', 'exact'],
|
||||
{'default': 'legacy_png'},
|
||||
],
|
||||
'transfer_quality': ['INT', {'default': 90, 'min': 1, 'max': 100, 'step': 1}],
|
||||
@@ -45,7 +45,8 @@ ADDED_OPTIONAL_INPUTS = {
|
||||
|
||||
# Nodes added after the V1 baseline; not part of saved-workflow contracts.
|
||||
# DistributedWorkerSink is internal (dev-only) and only inserted by the master.
|
||||
ADDED_NODES = {'DistributedWorkerSink', 'DistributedImageListStart', 'DistributedImageListEnd', 'DistributedMediaCollector'}
|
||||
ADDED_NODES = {'DistributedWorkerSink', 'DistributedImageListStart', 'DistributedImageListEnd',
|
||||
'DistributedImageListGetValue'}
|
||||
|
||||
|
||||
def normalized_input(value):
|
||||
@@ -273,7 +274,7 @@ async def check_image_list_lazy_executor(mapping, module, prompt_server, asset_m
|
||||
original_contexts = set(runtime._MASTER_JOBS)
|
||||
image_list_nodes = sys.modules[module.__name__ + '.nodes.image_list']
|
||||
workflow_meta = {'nodes': [{'id': 2, 'type': 'DistributedImageListStart'}], 'extra': {'acceptance': True}}
|
||||
async def fake_worker_results(_context_id, *, model_callback=None):
|
||||
async def fake_worker_results(_context_id, *, model_callback=None, include_values=False):
|
||||
state = runtime.get_master_job(_context_id)
|
||||
assert state.prompt_id == 'image-list-lazy', state.prompt_id
|
||||
assert state.workflow_meta == workflow_meta, state.workflow_meta
|
||||
@@ -281,6 +282,7 @@ async def check_image_list_lazy_executor(mapping, module, prompt_server, asset_m
|
||||
return (
|
||||
[torch.full((1, 8, 8, 3), value, dtype=torch.float32) for value in (0.125, 0.5, 0.875)],
|
||||
[None, None, None], [None, None, None],
|
||||
[{}, {}, {}],
|
||||
)
|
||||
|
||||
graph = {
|
||||
@@ -366,14 +368,16 @@ async def main():
|
||||
extension = await module.comfy_entrypoint()
|
||||
classes = await extension.get_node_list()
|
||||
mapping = {cls.GET_SCHEMA().node_id: cls for cls in classes}
|
||||
assert len(classes) == len(mapping) == len(BASELINE) + len(ADDED_NODES)
|
||||
assert len(classes) == len(mapping) == len(BASELINE) + len(ADDED_NODES) == 12
|
||||
assert set(mapping) == set(BASELINE) | ADDED_NODES
|
||||
for node_id in ADDED_NODES:
|
||||
assert nodes.NODE_CLASS_MAPPINGS[node_id] is mapping[node_id]
|
||||
media_info = mapping['DistributedMediaCollector'].GET_NODE_INFO_V1()
|
||||
assert media_info['output'] == ['IMAGE', 'AUDIO', 'MASK', 'STRING', 'VIDEO'], media_info
|
||||
assert all(media_info['output_is_list']), media_info
|
||||
assert media_info['input']['optional']['texts'][1]['forceInput'] is True
|
||||
assert 'DistributedMediaCollector' not in mapping
|
||||
collector_info = mapping['DistributedCollector'].GET_NODE_INFO_V1()
|
||||
assert collector_info['output'] == ['IMAGE', 'AUDIO'], collector_info
|
||||
assert not any(collector_info['output_is_list']), collector_info
|
||||
assert list(collector_info['input']['optional']) == [
|
||||
'images', 'audio', 'transfer_format', 'transfer_quality']
|
||||
sink_info = mapping['DistributedWorkerSink'].GET_NODE_INFO_V1()
|
||||
assert sink_info['output_node'] and sink_info['dev_only'], sink_info
|
||||
for node_id, cls in mapping.items():
|
||||
@@ -427,6 +431,42 @@ async def main():
|
||||
assert start_schema.display_name == 'Distributed Image List Start'
|
||||
assert end_schema.node_id == 'DistributedImageListEnd'
|
||||
assert end_schema.display_name == 'Distributed Image List End'
|
||||
runtime = sys.modules[module.__name__ + '.image_list.runtime']
|
||||
assert [input.id for input in end_schema.inputs] == [
|
||||
'images', 'masks', 'texts', 'context', 'transfer_format', 'transfer_quality'] + list(runtime.VALUE_NAMES)
|
||||
format_input, quality_input = end_schema.inputs[4:6]
|
||||
assert format_input.options == list(runtime.IMAGE_TRANSFER_FORMATS)
|
||||
assert format_input.default == 'exact' and quality_input.default == 90
|
||||
assert format_input.optional and quality_input.optional
|
||||
assert [output.io_type for output in end_schema.outputs] == [
|
||||
'IMAGE', 'MASK', 'STRING', 'DISTRIBUTED_IMAGE_LIST_VALUES']
|
||||
assert end_schema.outputs[3].id == end_schema.outputs[3].display_name == 'values'
|
||||
assert all(output.is_output_list for output in end_schema.outputs)
|
||||
for override in end_schema.inputs[6:]:
|
||||
assert override.optional and override.lazy
|
||||
assert override.get_io_type() == 'IMAGE,MASK,STRING,INT,FLOAT,BOOLEAN'
|
||||
getter = mapping['DistributedImageListGetValue']
|
||||
getter_schema = getter.GET_SCHEMA()
|
||||
assert getter_schema.display_name == 'Distributed Image List Get Value'
|
||||
assert not getter_schema.is_input_list
|
||||
assert getter_schema.outputs[0].id == getter_schema.outputs[0].display_name == 'value'
|
||||
assert getter_schema.outputs[0].io_type == '*'
|
||||
assert getter_schema.inputs[1].options == list(runtime.VALUE_NAMES)
|
||||
assert getter_schema.inputs[1].default == 'value_1'
|
||||
# Real core maps scalar mappings, never arbitrary tensor batches.
|
||||
tensor = torch.ones(3, 2, 2, 3)
|
||||
data, missing, hidden = execution.get_input_data({'value_name': 'value_1'}, getter, 'get-value')
|
||||
data['values'] = [{'value_1': tensor}, {'value_1': 17}, {'value_1': False}]
|
||||
mapped = await execution._async_map_node_over_list('get-value', 'get-value', getter, data,
|
||||
getter.FUNCTION, v3_data=hidden)
|
||||
mapped = await execution.resolve_map_node_over_list_results(mapped)
|
||||
assert len(mapped) == 3 and mapped[0].result[0] is tensor
|
||||
assert mapped[1].result == (17,) and mapped[2].result == (False,)
|
||||
try:
|
||||
getter.execute({}, 'value_1')
|
||||
raise AssertionError('missing Get Value key accepted')
|
||||
except ValueError as exc:
|
||||
assert 'absent' in str(exc)
|
||||
texts_input = next(input for input in end_schema.inputs if input.id == 'texts')
|
||||
texts_schema = texts_input.as_dict()
|
||||
assert texts_schema.get('forceInput') is True, texts_schema
|
||||
@@ -470,7 +510,6 @@ async def main():
|
||||
rows = runtime_mod.normalize_image_list_inputs(prepared['images'], None, seen['values'])
|
||||
assert [row['values'] for row in rows] == [{'value_1': 'shared', 'value_3': n} for n in (4, 5, 6)]
|
||||
print('IMAGE_LIST_AUTOGROW_OK real parser; value_N slots shared/per-item')
|
||||
assert [out.io_type for out in end_schema.outputs] == ['IMAGE', 'MASK', 'STRING']
|
||||
assert all(out.is_output_list for out in end_schema.outputs)
|
||||
assert end_schema.is_input_list and end_schema.is_output_node
|
||||
print('IMAGE_LIST_SCHEMA_OK exact IDs, names, typed list outputs')
|
||||
|
||||
@@ -0,0 +1,524 @@
|
||||
"""Exact Collector transport through real serializers, HTTP, route and queue.
|
||||
|
||||
Only ComfyUI's process-wide server/model-management and unrelated orchestration
|
||||
are replaced. Tensor/audio codecs, multipart bodies and aiohttp are real; no
|
||||
running ComfyUI service, GPU, provider or external network access is required.
|
||||
"""
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PKG = "dist_collector_exact_testpkg"
|
||||
JOB_ID = "exact-job"
|
||||
WORKER_ID = "worker-1"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def transport(monkeypatch):
|
||||
# Other legacy test loaders may leave aiohttp stubs behind. Replace only
|
||||
# those entries, never unload/reload real extension-backed dependencies.
|
||||
for name, loaded_module in list(sys.modules.items()):
|
||||
if (name == "aiohttp" or name.startswith("aiohttp.")) and not getattr(loaded_module, "__file__", None):
|
||||
monkeypatch.delitem(sys.modules, name)
|
||||
aiohttp = importlib.import_module("aiohttp")
|
||||
web = importlib.import_module("aiohttp.web")
|
||||
|
||||
def module(name, **attrs):
|
||||
result = types.ModuleType(name)
|
||||
result.__dict__.update(attrs)
|
||||
monkeypatch.setitem(sys.modules, name, result)
|
||||
return result
|
||||
|
||||
def load(relative):
|
||||
name = f"{PKG}.{relative}"
|
||||
spec = importlib.util.spec_from_file_location(name, ROOT.joinpath(*relative.split(".")).with_suffix(".py"))
|
||||
assert spec is not None and spec.loader is not None
|
||||
result = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, name, result)
|
||||
spec.loader.exec_module(result)
|
||||
parent, _, child = name.rpartition(".")
|
||||
setattr(sys.modules[parent], child, result)
|
||||
return result
|
||||
|
||||
module(PKG, __path__=[str(ROOT)])
|
||||
for package in ("nodes", "api", "utils", "serverless_bridge"):
|
||||
module(f"{PKG}.{package}", __path__=[str(ROOT / package)])
|
||||
server = types.SimpleNamespace(routes=web.RouteTableDef(), distributed_pending_jobs={})
|
||||
module("server", PromptServer=types.SimpleNamespace(instance=server))
|
||||
mm = module("comfy.model_management", throw_exception_if_processing_interrupted=lambda: None)
|
||||
progress = module("comfy.utils", ProgressBar=lambda _total: types.SimpleNamespace(update=lambda _n: None))
|
||||
module("comfy", __path__=[], model_management=mm, utils=progress)
|
||||
module(f"{PKG}.utils.logging", debug_log=lambda *_a, **_kw: None, log=lambda *_a, **_kw: None)
|
||||
module(f"{PKG}.utils.config", get_worker_timeout_seconds=lambda: 0.1,
|
||||
load_config=lambda: {"workers": []}, is_master_delegate_only=lambda: False)
|
||||
|
||||
async def handle_api_error(_request, error, status=500):
|
||||
return web.json_response({"error": str(error)}, status=status)
|
||||
|
||||
module(f"{PKG}.utils.network", handle_api_error=handle_api_error,
|
||||
get_client_session=AsyncMock(), build_worker_url=lambda _worker: "http://unused.invalid",
|
||||
probe_worker=AsyncMock(return_value=None))
|
||||
module(f"{PKG}.utils.async_helpers", run_async_in_server_loop=asyncio.run)
|
||||
module(f"{PKG}.api.queue_orchestration", ensure_distributed_state=lambda: None,
|
||||
orchestrate_distributed_execution=AsyncMock())
|
||||
module(f"{PKG}.api.queue_request", parse_queue_request_payload=lambda value: value)
|
||||
for name in ("utils.constants", "utils.image", "utils.audio_payload", "utils.transfer_codec"):
|
||||
load(name)
|
||||
bridge_codec = load("serverless_bridge.codec")
|
||||
load("serverless_bridge.image_transfer")
|
||||
image_codec = load("serverless_bridge.image_codec")
|
||||
collector = load("nodes.collector")
|
||||
routes = load("api.job_routes")
|
||||
return types.SimpleNamespace(aiohttp=aiohttp, web=web, server=server,
|
||||
collector=collector, routes=routes, bridge_codec=bridge_codec,
|
||||
image_codec=image_codec)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def endpoint(transport, monkeypatch, *, request_limit=2 * 1024 * 1024,
|
||||
chunk_target=1024 * 1024, client_max_size=None, handler=None):
|
||||
"""A disposable loopback endpoint; record actual HTTP lengths, not estimates."""
|
||||
queue = asyncio.Queue()
|
||||
transport.server.distributed_jobs_lock = asyncio.Lock()
|
||||
transport.server.distributed_pending_jobs = {JOB_ID: queue}
|
||||
requests = []
|
||||
web = transport.web
|
||||
|
||||
@web.middleware
|
||||
async def record_request(request, handler):
|
||||
record = {"bytes": request.content_length, "content_type": request.content_type}
|
||||
requests.append(record)
|
||||
response = await handler(request)
|
||||
record["status"] = response.status
|
||||
if response.status < 400 and request.content_type.startswith("multipart/"):
|
||||
form = await request.post()
|
||||
record["envelope"] = json.loads(form["envelope"])
|
||||
return response
|
||||
|
||||
app = web.Application(client_max_size=client_max_size or request_limit, middlewares=[record_request])
|
||||
app.router.add_post("/distributed/job_complete", handler or transport.routes.job_complete_endpoint)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
try:
|
||||
site = web.TCPSite(runner, "127.0.0.1", 0)
|
||||
await site.start()
|
||||
port = site._server.sockets[0].getsockname()[1]
|
||||
async with transport.aiohttp.ClientSession() as session:
|
||||
monkeypatch.setattr(transport.collector, "get_client_session", AsyncMock(return_value=session))
|
||||
monkeypatch.setattr(transport.collector, "TRANSFER_MAX_REQUEST_BYTES", request_limit)
|
||||
monkeypatch.setattr(transport.collector, "TRANSFER_MAX_CHUNK_BYTES", chunk_target)
|
||||
yield types.SimpleNamespace(
|
||||
session=session, url=f"http://127.0.0.1:{port}", requests=requests, queue=queue,
|
||||
node=transport.collector.DistributedCollectorNode(),
|
||||
)
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
|
||||
def frames(dtype=torch.float32, channels=3, count=5, height=7, width=9):
|
||||
# Non-contiguous, out-of-range and non-8-bit-representable values exercise
|
||||
# detach/contiguous without permitting clipping, alpha loss or quantization.
|
||||
values = torch.arange(count * height * width * channels, dtype=torch.float64)
|
||||
values = (values.remainder(41) - 11) / 17
|
||||
images = values.reshape(count, width, height, channels).transpose(1, 2).to(dtype)
|
||||
images[0, 0, 0, 0] = -0.0
|
||||
return images.requires_grad_(True)
|
||||
|
||||
|
||||
def assert_exact(actual, expected):
|
||||
assert actual.dtype == expected.dtype
|
||||
assert actual.shape == expected.shape
|
||||
assert actual.device.type == "cpu"
|
||||
assert actual.is_contiguous()
|
||||
# Compare raw bits as well as values, including BF16 and signed zero.
|
||||
assert torch.equal(actual, expected)
|
||||
assert torch.equal(actual.view(torch.uint8), expected.detach().contiguous().view(torch.uint8))
|
||||
|
||||
|
||||
def queued_items(queue):
|
||||
return [queue.get_nowait() for _ in range(queue.qsize())]
|
||||
|
||||
|
||||
async def send(ctx, images, *, audio=None, fmt="exact"):
|
||||
await ctx.node.send_batch_to_master(images, audio, JOB_ID, ctx.url, WORKER_ID, fmt, 17)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("delegate_only", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32, torch.float64])
|
||||
@pytest.mark.parametrize("channels", [1, 3, 4])
|
||||
def test_exact_sender_route_and_collector_preserve_dtype_values_and_channels(
|
||||
transport, monkeypatch, dtype, channels, delegate_only):
|
||||
images = frames(dtype, channels)
|
||||
frame_bytes = images[0].numel() * images.element_size()
|
||||
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, request_limit=reserve + 2 * frame_bytes,
|
||||
chunk_target=2 * frame_bytes) as ctx:
|
||||
await send(ctx, images)
|
||||
items = queued_items(ctx.queue)
|
||||
assert [item["image_index"] for item in items] == list(range(len(images)))
|
||||
assert [item["is_last"] for item in items] == [False] * 4 + [True]
|
||||
assert all(item["worker_id"] == WORKER_ID and item["format"] == "exact" for item in items)
|
||||
for item, expected in zip(items, images.split(1)):
|
||||
assert_exact(item["tensor"], expected)
|
||||
await ctx.queue.put(item)
|
||||
envelopes = [record["envelope"] for record in ctx.requests]
|
||||
assert [(e["start_index"], e["count"]) for e in envelopes] == [(0, 2), (2, 2), (4, 1)]
|
||||
assert all(record["bytes"] <= reserve + 2 * frame_bytes for record in ctx.requests)
|
||||
master_images = None if delegate_only else images[:1].detach().clone().add(0.03125)
|
||||
actual, _audio = await ctx.node.execute(
|
||||
master_images, None, multi_job_id=JOB_ID, enabled_worker_ids=json.dumps([WORKER_ID]),
|
||||
delegate_only=delegate_only, transfer_format="exact",
|
||||
)
|
||||
expected = images if delegate_only else torch.cat([master_images, images])
|
||||
assert_exact(actual, expected)
|
||||
assert JOB_ID not in transport.server.distributed_pending_jobs
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_single_frame_can_exceed_chunk_target_but_not_request_limit(transport, monkeypatch):
|
||||
images = frames(count=3)
|
||||
frame_bytes = images[0].numel() * images.element_size()
|
||||
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, request_limit=reserve + frame_bytes,
|
||||
chunk_target=frame_bytes // 2) as ctx:
|
||||
await send(ctx, images)
|
||||
assert [record["envelope"]["count"] for record in ctx.requests] == [1, 1, 1]
|
||||
assert all(record["bytes"] <= reserve + frame_bytes for record in ctx.requests)
|
||||
assert_exact(torch.cat([item["tensor"] for item in queued_items(ctx.queue)]), images)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_oversized_frame_is_rejected_before_encoding_or_post(transport, monkeypatch):
|
||||
images = frames(count=3)
|
||||
frame_bytes = images[0].numel() * images.element_size()
|
||||
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
|
||||
encode = AsyncMock(side_effect=AssertionError("oversized frame must not be encoded"))
|
||||
monkeypatch.setattr(transport.collector, "_encode_exact_frames", encode)
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, request_limit=reserve + frame_bytes - 1) as ctx:
|
||||
with pytest.raises(transport.collector.transfer_codec.FrameTooLargeError, match="request limit"):
|
||||
await send(ctx, images)
|
||||
assert ctx.requests == []
|
||||
assert ctx.queue.empty()
|
||||
encode.assert_not_called()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(0, 2, 3, 3), (1, 0, 3, 3), (1, 2, 0, 3), (1, 2, 3, 0),
|
||||
(1, 2, 3, 2), (1, 2, 3, 5), (1, 2, 3), (2, 3)])
|
||||
def test_exact_sender_rejects_invalid_geometry_before_post(transport, monkeypatch, shape):
|
||||
image = torch.empty(shape)
|
||||
with pytest.raises(transport.collector.transfer_codec.TransferCodecError):
|
||||
transport.collector._exact_frame_bytes(image)
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch) as ctx:
|
||||
# An empty batch is also a no-media completion error (ValueError).
|
||||
with pytest.raises(ValueError):
|
||||
await send(ctx, image)
|
||||
assert ctx.requests == []
|
||||
assert ctx.queue.empty()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.uint8, torch.int32, torch.bool, torch.complex64])
|
||||
def test_exact_sender_rejects_unsupported_dtype_before_post(transport, monkeypatch, dtype):
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch) as ctx:
|
||||
with pytest.raises(transport.collector.transfer_codec.TransferCodecError, match="dtype"):
|
||||
await send(ctx, torch.zeros(1, 2, 3, 3, dtype=dtype))
|
||||
assert ctx.requests == []
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_frame_planning_enforces_bridge_byte_limit(transport, monkeypatch):
|
||||
monkeypatch.setattr(transport.bridge_codec, "MAX_PAYLOAD_BYTES", 32)
|
||||
with pytest.raises(transport.collector.transfer_codec.TransferCodecError, match="[Bb]yte|limit"):
|
||||
transport.collector._exact_frame_bytes(torch.zeros(1, 2, 3, 3))
|
||||
|
||||
|
||||
def test_exact_frame_planning_rejects_non_dense_layout(transport):
|
||||
image = torch.zeros(1, 2, 3, 3).to_sparse()
|
||||
with pytest.raises(transport.collector.transfer_codec.TransferCodecError, match="dense|layout|strided"):
|
||||
transport.collector._exact_frame_bytes(image)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")])
|
||||
def test_exact_sender_rejects_nonfinite_payload(transport, monkeypatch, value):
|
||||
images = torch.full((1, 2, 3, 3), value)
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch) as ctx:
|
||||
with pytest.raises(transport.collector.transfer_codec.TransferCodecError, match="finite"):
|
||||
await send(ctx, images)
|
||||
assert ctx.requests == []
|
||||
assert ctx.queue.empty()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def exact_chunk(transport, count=2):
|
||||
images = frames(count=count).detach()
|
||||
encoded = [transport.bridge_codec.encode_payload("IMAGE", frame) for frame in images.split(1)]
|
||||
return {
|
||||
"job_id": JOB_ID, "worker_id": WORKER_ID, "format": "exact",
|
||||
"start_index": 0, "count": count, "width": images.shape[2], "height": images.shape[1],
|
||||
"is_last": True, "descriptors": [descriptor for descriptor, _ in encoded],
|
||||
}, [payload for _, payload in encoded]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", [
|
||||
"zero_count", "bool_width", "negative_height", "negative_index", "bool_index",
|
||||
"missing_descriptor", "extra_descriptor", "missing_part", "extra_part", "not_a_descriptor",
|
||||
"wrong_height", "wrong_batch", "bad_channels", "bad_dtype", "bad_layout", "bad_encoding",
|
||||
"extra_descriptor_field", "truncated", "trailing_bytes", "nonfinite", "compressed_spoof",
|
||||
])
|
||||
def test_exact_route_rejects_bad_geometry_and_payload_atomically(transport, monkeypatch, case):
|
||||
envelope, parts = exact_chunk(transport)
|
||||
descriptor = envelope["descriptors"][-1]
|
||||
if case == "zero_count":
|
||||
envelope["count"] = 0
|
||||
elif case == "bool_width":
|
||||
envelope["width"] = True
|
||||
elif case == "negative_height":
|
||||
envelope["height"] = -1
|
||||
elif case == "negative_index":
|
||||
envelope["start_index"] = -1
|
||||
elif case == "bool_index":
|
||||
envelope["start_index"] = True
|
||||
elif case == "missing_descriptor":
|
||||
envelope["descriptors"].pop()
|
||||
elif case == "extra_descriptor":
|
||||
envelope["descriptors"].append(dict(descriptor))
|
||||
elif case == "missing_part":
|
||||
parts.pop()
|
||||
elif case == "extra_part":
|
||||
parts.append(parts[-1])
|
||||
elif case == "not_a_descriptor":
|
||||
envelope["descriptors"][-1] = None
|
||||
elif case == "wrong_height":
|
||||
descriptor["shape"][1] += 1
|
||||
elif case == "wrong_batch":
|
||||
descriptor["shape"][0] = 2
|
||||
elif case == "bad_channels":
|
||||
descriptor["shape"][-1] = 2
|
||||
elif case == "bad_dtype":
|
||||
descriptor["dtype"] = "uint8"
|
||||
elif case == "bad_layout":
|
||||
descriptor["layout"] = "BCHW"
|
||||
elif case == "bad_encoding":
|
||||
descriptor["encoding"] = "pickle"
|
||||
elif case == "extra_descriptor_field":
|
||||
descriptor["unknown"] = True
|
||||
elif case == "truncated":
|
||||
parts[-1] = parts[-1][:-1]
|
||||
elif case == "trailing_bytes":
|
||||
parts[-1] += b"\0"
|
||||
elif case == "nonfinite":
|
||||
bad = torch.full(tuple(descriptor["shape"]), float("nan"))
|
||||
parts[-1] = bad.numpy().tobytes()
|
||||
elif case == "compressed_spoof":
|
||||
# A valid serverless image-transfer descriptor must NOT be accepted as
|
||||
# exact by the direct route, even though the shared bridge can decode it.
|
||||
envelope["descriptors"][-1], parts[-1] = transport.image_codec.encode_image(
|
||||
frames(count=1).detach(), transfer_format="png", transfer_quality=90,
|
||||
)
|
||||
# Prove the spoof is a valid compressed bridge payload, not simply junk.
|
||||
decoded = transport.bridge_codec.decode_payload("IMAGE", envelope["descriptors"][-1], parts[-1])
|
||||
assert list(decoded.shape) == descriptor["shape"]
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch) as ctx:
|
||||
async with ctx.session.post(
|
||||
ctx.url + "/distributed/job_complete",
|
||||
data=ctx.node._build_multipart(envelope, parts),
|
||||
) as response:
|
||||
body = await response.text()
|
||||
assert response.status == 400, body
|
||||
# A corrupt second frame must not leave the first in the queue.
|
||||
assert ctx.queue.empty()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,body", [(400, "Invalid JSON payload: old master"),
|
||||
(413, "upload limit"), (500, "unexpected failure")])
|
||||
def test_exact_master_rejection_never_falls_back(transport, monkeypatch, status, body):
|
||||
async def reject(_request):
|
||||
return transport.web.Response(status=status, text=body)
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, handler=reject, chunk_target=1) as ctx:
|
||||
legacy = AsyncMock(side_effect=AssertionError("exact must never use legacy transport"))
|
||||
compressed = AsyncMock(side_effect=AssertionError("exact must never use compressed transport"))
|
||||
monkeypatch.setattr(ctx.node, "_send_legacy_batch", legacy)
|
||||
monkeypatch.setattr(ctx.node, "_send_compressed_batch", compressed)
|
||||
with pytest.raises(RuntimeError, match=f"HTTP {status}") as raised:
|
||||
await send(ctx, frames(count=3))
|
||||
assert "never" in str(raised.value) and "8-bit" in str(raised.value)
|
||||
assert len(ctx.requests) == 1
|
||||
assert ctx.requests[0]["content_type"].startswith("multipart/")
|
||||
assert ctx.queue.empty()
|
||||
legacy.assert_not_called()
|
||||
compressed.assert_not_called()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fmt", ["legacy_png", "png"])
|
||||
def test_exact_collector_rejects_old_worker_8bit_result(transport, monkeypatch, fmt):
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch) as ctx:
|
||||
await send(ctx, frames(count=1).detach(), fmt=fmt)
|
||||
assert ctx.queue.qsize() == 1
|
||||
with pytest.raises(RuntimeError, match="requested exact"):
|
||||
await ctx.node.execute(
|
||||
None, None, multi_job_id=JOB_ID, enabled_worker_ids=json.dumps([WORKER_ID]),
|
||||
delegate_only=True, transfer_format="exact",
|
||||
)
|
||||
assert JOB_ID not in transport.server.distributed_pending_jobs
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_collector_shape_mismatch_raises_instead_of_returning_local_images(transport, monkeypatch):
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch) as ctx:
|
||||
await send(ctx, frames(count=1, channels=4))
|
||||
with pytest.raises(RuntimeError, match="shape|combin|mismatch"):
|
||||
await ctx.node.execute(
|
||||
frames(count=1, channels=3), None, multi_job_id=JOB_ID,
|
||||
enabled_worker_ids=json.dumps([WORKER_ID]), transfer_format="exact",
|
||||
)
|
||||
assert JOB_ID not in transport.server.distributed_pending_jobs
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_incomplete_worker_raises_and_cleans_queue(transport, monkeypatch):
|
||||
"""A timed-out worker's first frame is not a successful partial result."""
|
||||
envelope, parts = exact_chunk(transport, count=1)
|
||||
envelope["is_last"] = False
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch) as ctx:
|
||||
async with ctx.session.post(
|
||||
ctx.url + "/distributed/job_complete", data=ctx.node._build_multipart(envelope, parts),
|
||||
) as response:
|
||||
assert response.status == 200, await response.text()
|
||||
assert ctx.queue.qsize() == 1
|
||||
with pytest.raises(RuntimeError, match="Exact transfer incomplete"):
|
||||
await asyncio.wait_for(ctx.node.execute(
|
||||
frames(count=1).detach(), None, multi_job_id=JOB_ID,
|
||||
enabled_worker_ids=json.dumps([WORKER_ID]), transfer_format="exact",
|
||||
), timeout=2)
|
||||
assert JOB_ID not in transport.server.distributed_pending_jobs
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_oversized_multipart_metadata_is_rejected_before_post(transport, monkeypatch):
|
||||
images = frames(count=1)
|
||||
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
|
||||
limit = reserve + images.numel() * images.element_size()
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, request_limit=limit) as ctx:
|
||||
with pytest.raises(transport.collector.transfer_codec.FrameTooLargeError, match="request limit"):
|
||||
await ctx.node.send_batch_to_master(
|
||||
images, None, JOB_ID, ctx.url, "w" * limit, "exact", 90,
|
||||
)
|
||||
assert ctx.requests == []
|
||||
assert ctx.queue.empty()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_actual_server_upload_rejection_is_413_without_retry(transport, monkeypatch):
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, client_max_size=1024) as ctx:
|
||||
with pytest.raises(RuntimeError, match="HTTP 413"):
|
||||
await send(ctx, frames(count=3, height=64, width=64))
|
||||
assert len(ctx.requests) == 1
|
||||
assert ctx.requests[0]["status"] == 413
|
||||
assert ctx.queue.empty()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_oversized_audio_is_rejected_before_any_image_post(transport, monkeypatch):
|
||||
images = frames(count=3)
|
||||
audio = {"waveform": torch.zeros(1, 1, 100000), "sample_rate": 48000}
|
||||
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, request_limit=reserve + 1024) as ctx:
|
||||
with pytest.raises(transport.collector.transfer_codec.FrameTooLargeError, match="audio.*request limit"):
|
||||
await send(ctx, images, audio=audio)
|
||||
assert ctx.requests == []
|
||||
assert ctx.queue.empty()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_exact_tiny_frames_cannot_overflow_multipart_metadata_budget(transport, monkeypatch):
|
||||
images = torch.full((2048, 1, 1, 1), 0.125, dtype=torch.float16)
|
||||
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
|
||||
limit = reserve + images.numel() * images.element_size()
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, request_limit=limit, chunk_target=limit) as ctx:
|
||||
await send(ctx, images)
|
||||
assert len(ctx.requests) > 1
|
||||
assert all(record["bytes"] <= limit for record in ctx.requests)
|
||||
assert_exact(torch.cat([item["tensor"] for item in queued_items(ctx.queue)]), images)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("separate", [False, True])
|
||||
def test_exact_audio_is_preserved_and_only_final_completion_is_terminal(transport, monkeypatch, separate):
|
||||
images = frames(count=3)
|
||||
audio = {"waveform": torch.linspace(-0.9, 0.9, 60).reshape(1, 1, -1), "sample_rate": 48000}
|
||||
frame_bytes = images[0].numel() * images.element_size()
|
||||
reserve = transport.collector.DistributedCollectorNode.TRANSFER_ENVELOPE_RESERVE_BYTES
|
||||
media_budget = frame_bytes if separate else frame_bytes + 8192
|
||||
limit = reserve + media_budget
|
||||
|
||||
async def scenario():
|
||||
async with endpoint(transport, monkeypatch, request_limit=limit, chunk_target=frame_bytes) as ctx:
|
||||
await send(ctx, images, audio=audio)
|
||||
items = queued_items(ctx.queue)
|
||||
assert len(ctx.requests) == 3 + int(separate)
|
||||
assert [item["is_last"] for item in items] == [False] * (len(items) - 1) + [True]
|
||||
assert all(item["audio"] is None for item in items[:-1])
|
||||
assert items[-1]["audio"]["sample_rate"] == audio["sample_rate"]
|
||||
assert_exact(items[-1]["audio"]["waveform"], audio["waveform"])
|
||||
assert_exact(torch.cat([item["tensor"] for item in items if item["tensor"] is not None]), images)
|
||||
assert all(record["bytes"] <= limit for record in ctx.requests)
|
||||
if separate:
|
||||
assert items[-1]["tensor"] is None
|
||||
assert ctx.requests[-1]["content_type"] == "application/json"
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Real Collector aggregation of typed bridge deliveries and direct results.
|
||||
|
||||
Only Comfy's host services are stubbed by the shared Collector loader. Tensor
|
||||
normalization, participant ordering, queue consumption and cleanup are real.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import types
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from test_collector_list_inputs import _load_collector_module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collector_module():
|
||||
module = _load_collector_module()
|
||||
module.prompt_server.distributed_jobs_lock = asyncio.Lock()
|
||||
module.prompt_server.distributed_serverless = types.SimpleNamespace(cancel=AsyncMock())
|
||||
return module
|
||||
|
||||
|
||||
def native_media(value=0):
|
||||
image = torch.arange(24, dtype=torch.float64).reshape(1, 2, 3, 4) / 997 + value
|
||||
image[0, 0, 0, 0] = -0.0
|
||||
audio = {"waveform": torch.arange(6, dtype=torch.float64).reshape(1, 2, 3) / 997 + value,
|
||||
"sample_rate": 24000}
|
||||
return {"images": image, "audio": audio}
|
||||
|
||||
|
||||
def assert_exact(actual, expected):
|
||||
assert actual.dtype == expected.dtype and actual.shape == expected.shape
|
||||
assert torch.equal(actual.contiguous().view(torch.uint8), expected.contiguous().view(torch.uint8))
|
||||
|
||||
|
||||
def collect_queue(module, messages, participants, *, local=None, delegate=True):
|
||||
queue = asyncio.Queue()
|
||||
for message in messages:
|
||||
queue.put_nowait(message)
|
||||
module.prompt_server.distributed_pending_jobs["typed-job"] = queue
|
||||
values = local or {"images": None, "audio": None}
|
||||
return asyncio.run(asyncio.wait_for(module.DistributedCollectorNode().execute(
|
||||
**values, multi_job_id="typed-job", enabled_worker_ids=json.dumps(participants),
|
||||
delegate_only=delegate), timeout=2))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("delegate", [False, True])
|
||||
@pytest.mark.parametrize("socket_set", [("images",), ("audio",), ("images", "audio")])
|
||||
def test_typed_and_direct_results_preserve_batch_and_audio_order(collector_module, delegate, socket_set):
|
||||
module = collector_module
|
||||
local, typed_a, typed_b, direct = [native_media(n) for n in range(4)]
|
||||
typed = {socket: [typed_a[socket], typed_b[socket]] for socket in socket_set}
|
||||
direct_message = {"worker_id": "direct", "is_last": True}
|
||||
if "images" in socket_set:
|
||||
direct_message.update(tensor=direct["images"], image_index=0)
|
||||
if "audio" in socket_set:
|
||||
direct_message["audio"] = direct["audio"]
|
||||
messages = [direct_message, {"worker_id": "serverless:pool:1", "media": typed, "is_last": True}]
|
||||
# Delivery order differs from participant order. Duplicate configured IDs
|
||||
# must not duplicate a participant's data or leave collection waiting.
|
||||
participants = ["serverless:pool:1", "direct", "serverless:pool:1"]
|
||||
local_inputs = {socket: local[socket] if socket in socket_set else None for socket in ("images", "audio")}
|
||||
images, audio = collect_queue(module, messages, participants, local=local_inputs, delegate=delegate)
|
||||
expected = ([] if delegate else [local]) + [typed_a, typed_b, direct]
|
||||
if "images" in socket_set:
|
||||
assert_exact(images, torch.cat([row["images"] for row in expected]))
|
||||
else:
|
||||
assert images is None
|
||||
if "audio" in socket_set:
|
||||
assert audio["sample_rate"] == 24000
|
||||
assert_exact(audio["waveform"], torch.cat([row["audio"]["waveform"] for row in expected], dim=-1))
|
||||
else:
|
||||
assert torch.equal(audio["waveform"], module.DistributedCollectorNode.EMPTY_AUDIO["waveform"])
|
||||
assert module.prompt_server.distributed_pending_jobs == {}
|
||||
module.prompt_server.distributed_serverless.cancel.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("media", [{}, {"texts": ["not a Collector socket"]}, {"images": []},
|
||||
{"audio": [None]}, {"images": "not a list"},
|
||||
{"images": [torch.zeros(2, 2, 3)]}])
|
||||
def test_invalid_typed_delivery_cancels_and_removes_queue(collector_module, media):
|
||||
module = collector_module
|
||||
with pytest.raises(RuntimeError, match="media|items|IMAGE"):
|
||||
collect_queue(module, [{"worker_id": "serverless:pool:1", "media": media, "is_last": True}],
|
||||
["serverless:pool:1"])
|
||||
assert module.prompt_server.distributed_pending_jobs == {}
|
||||
module.prompt_server.distributed_serverless.cancel.assert_awaited_once_with(collector_job_id="typed-job")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["nonfinal", "unexpected", "duplicate", "provider-error", "shape"])
|
||||
def test_typed_delivery_protocol_failures_are_not_partial_success(collector_module, failure):
|
||||
media = {"images": [native_media()["images"]]}
|
||||
first = {"worker_id": "serverless:pool:1", "media": media, "is_last": True}
|
||||
messages = [first]
|
||||
participants = ["serverless:pool:1", "serverless:pool:2"]
|
||||
if failure == "nonfinal":
|
||||
first["is_last"] = False
|
||||
elif failure == "unexpected":
|
||||
first["worker_id"] = "other"
|
||||
elif failure == "duplicate":
|
||||
messages.append(dict(first))
|
||||
elif failure == "provider-error":
|
||||
messages.append({"worker_id": "serverless:pool:2", "serverless_error": "offline failure", "is_last": True})
|
||||
else:
|
||||
messages.append({"worker_id": "serverless:pool:2", "media": {"images": [torch.zeros(1, 4, 5, 3)]},
|
||||
"is_last": True})
|
||||
with pytest.raises(RuntimeError, match="atomic|Unexpected|Duplicate|failed|shape mismatch"):
|
||||
collect_queue(collector_module, messages, participants)
|
||||
assert collector_module.prompt_server.distributed_pending_jobs == {}
|
||||
collector_module.prompt_server.distributed_serverless.cancel.assert_awaited_once_with(collector_job_id="typed-job")
|
||||
@@ -472,6 +472,25 @@ class DispatchSelectionTests(unittest.IsolatedAsyncioTestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, "execution_targets.*upgrade"):
|
||||
await dispatch.preflight_image_list_workers([{"id": "worker"}])
|
||||
|
||||
async def test_image_list_value_results_requires_exact_true_only_for_overrides(self):
|
||||
for value in (None, False, 1, "true", [], True):
|
||||
capability = _valid_capability()
|
||||
if value is not None:
|
||||
capability['value_results'] = value
|
||||
session = _FakeSession({('worker', 'capabilities'): (200, capability),
|
||||
('worker', 'queue'): (200, _valid_queue())})
|
||||
async def fake_session():
|
||||
return session
|
||||
with patch.object(dispatch, 'get_client_session', side_effect=fake_session), patch.object(
|
||||
dispatch, 'build_worker_url', side_effect=lambda worker, path: f"http://{worker['id']}{path}"
|
||||
):
|
||||
self.assertEqual(await dispatch.preflight_image_list_workers([{'id': 'worker'}]), {'worker': True})
|
||||
if value is True:
|
||||
self.assertEqual(await dispatch.preflight_image_list_workers(
|
||||
[{'id': 'worker'}], require_value_results=True), {'worker': True})
|
||||
else:
|
||||
with self.assertRaisesRegex(RuntimeError, 'value_results.*upgrade'):
|
||||
await dispatch.preflight_image_list_workers([{'id': 'worker'}], require_value_results=True)
|
||||
async def test_image_list_preflight_fails_closed_for_malformed_capabilities(self):
|
||||
malformed = {"supported": True, "protocol_version": 2, "node_ids": None}
|
||||
session = _FakeSession({("worker", "capabilities"): (200, malformed)})
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Run the real framework's CPU-only private-cache retention regression."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_native_local_cache_retention():
|
||||
core = os.environ.get('COMFYUI_SOURCE_ROOT')
|
||||
if not core:
|
||||
pytest.skip('Set COMFYUI_SOURCE_ROOT for CPU-only real-framework validation')
|
||||
helper = Path(__file__).parent / 'helpers' / 'local_cache_retention_check.py'
|
||||
env = dict(os.environ, CUDA_VISIBLE_DEVICES='', HIP_VISIBLE_DEVICES='')
|
||||
env.pop('PYTHONPATH', None)
|
||||
result = subprocess.run([sys.executable, str(helper), core], env=env,
|
||||
text=True, capture_output=True, timeout=90)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert 'PASS: cross-run native cache reuse' in result.stdout
|
||||
@@ -191,6 +191,50 @@ class MasterRuntimeTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(result[2], [str(i) for i in range(9)])
|
||||
self.preflight.assert_awaited_once_with(self.workers, require_idle=False)
|
||||
|
||||
async def test_busy_direct_peer_does_not_block_selected_cloud_slots(self):
|
||||
self.workers.append({'id': 'worker', 'type': 'remote'})
|
||||
context = self.create(participates=False, count=3)
|
||||
self.config['workers'] = self.workers + [{'id': 'pool', 'type': 'serverless'}]
|
||||
self.state.worker_ids.append('pool')
|
||||
self.state.original_prompt['end']['inputs']['value_10'] = ['start', 2]
|
||||
fixture = self
|
||||
cloud_calls = []
|
||||
|
||||
async def busy_preflight(workers, **kwargs):
|
||||
if kwargs['require_idle']:
|
||||
raise RuntimeError('No selected feature-compatible direct worker is idle')
|
||||
return {}
|
||||
|
||||
self.preflight.side_effect = busy_preflight
|
||||
|
||||
class Controller:
|
||||
slots = [('serverless', 'pool', 0)]
|
||||
close = AsyncMock()
|
||||
|
||||
async def dispatch(self, slot, index):
|
||||
cloud_calls.append(index)
|
||||
return {'index': index}
|
||||
|
||||
def poll(self, inflight):
|
||||
slot = self.slots[0]
|
||||
if slot in inflight:
|
||||
index = inflight[slot]['index']
|
||||
return slot, index, {'image': fixture.items[index]['image'],
|
||||
'values': {'value_10': index}}
|
||||
return None
|
||||
|
||||
controller = Controller()
|
||||
create_slots = AsyncMock(return_value=controller)
|
||||
with patch.dict(sys.modules, {'_master_slot_tests.image_list.serverless':
|
||||
types.SimpleNamespace(create_serverless_slots=create_slots)}):
|
||||
result = await self.rt.run_image_list_job(context, include_values=True)
|
||||
self.assertEqual(cloud_calls, [0, 1, 2])
|
||||
self.assertEqual(result[3], [{'value_1': str(i), 'value_10': i} for i in range(3)])
|
||||
self.preflight.assert_awaited_once_with(self.workers, require_idle=False, require_value_results=True)
|
||||
self.network.get_client_session.assert_not_awaited()
|
||||
controller.close.assert_awaited_once()
|
||||
self.assertFalse(self.local_calls or self.dispatched or self.rt._MASTER_JOBS)
|
||||
|
||||
async def test_orchestrator_mode_never_instantiates_local_slot(self):
|
||||
self.workers.append({'id': 'worker', 'type': 'remote'})
|
||||
await self.rt.run_image_list_job(self.create(participates=False))
|
||||
@@ -199,6 +243,73 @@ class MasterRuntimeTests(unittest.IsolatedAsyncioTestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, 'worker|fallback'):
|
||||
self.rt.create_master_job('start', 'end', graph(), self.items, [], '')
|
||||
|
||||
async def test_values_reconstructed_in_source_order_without_transporting_originals(self):
|
||||
self.workers.append({'id': 'remote', 'type': 'remote'})
|
||||
context = self.create(count=6)
|
||||
tensor = torch.ones(4, 2, 2, 3, dtype=torch.float64)
|
||||
for index, item in enumerate(self.items):
|
||||
item['values'].update(value_2=tensor, value_3=True, value_4=index / 8)
|
||||
self.local_delay = 0.1 # Remote results arrive ahead of master item zero.
|
||||
with patch.object(self.rt, 'encode_result', side_effect=AssertionError('originals must not return')):
|
||||
result = await self.rt.run_image_list_job(context, include_values=True)
|
||||
self.assertFalse(self.rt._MASTER_JOBS)
|
||||
self.assertEqual(len(result), 4)
|
||||
for index, row in enumerate(result[3]):
|
||||
self.assertEqual(row['value_1'], str(index))
|
||||
self.assertIs(row['value_2'], tensor)
|
||||
self.assertIs(row['value_3'], True)
|
||||
self.assertEqual(row['value_4'], index / 8)
|
||||
self.assertTrue(self.dispatched and self.local_calls)
|
||||
|
||||
async def test_overrides_merge_add_and_recheck_capability_before_every_item(self):
|
||||
self.workers.append({'id': 'remote', 'type': 'remote'})
|
||||
context = self.create(participates=False, count=3)
|
||||
self.state.original_prompt['end']['inputs'].update(value_1=['start', 4], value_10=['start', 2])
|
||||
original_set = self.rt.set_master_result
|
||||
def override(context, index, token, result):
|
||||
result['values'] = {'value_1': f'new-{index}', 'value_10': index}
|
||||
return original_set(context, index, token, result)
|
||||
with patch.object(self.rt, 'set_master_result', side_effect=override):
|
||||
result = await self.rt.run_image_list_job(context, include_values=True)
|
||||
self.assertEqual(result[3], [{'value_1': f'new-{index}', 'value_10': index} for index in range(3)])
|
||||
self.assertEqual(self.preflight.await_count, 4)
|
||||
self.assertTrue(all(call.kwargs['require_value_results'] is True
|
||||
for call in self.preflight.await_args_list))
|
||||
self.assertEqual(self.items[0]['values'], {'value_1': '0'})
|
||||
|
||||
async def test_override_capability_downgrade_stops_before_upload(self):
|
||||
self.workers.append({'id': 'remote', 'type': 'remote'})
|
||||
context = self.create(participates=False)
|
||||
self.state.original_prompt['end']['inputs']['value_1'] = ['start', 4]
|
||||
self.preflight.side_effect = [{'remote': True}, RuntimeError('value_results unsupported')]
|
||||
with patch.object(self, 'post', wraps=self.post) as upload:
|
||||
with self.assertRaisesRegex(RuntimeError, 'value_results'):
|
||||
await self.rt.run_image_list_job(context, include_values=True)
|
||||
upload.assert_not_called()
|
||||
self.assertFalse(self.dispatched or self.rt._MASTER_JOBS)
|
||||
|
||||
async def test_missing_expected_override_fails_without_partial_output(self):
|
||||
self.workers.append({'id': 'remote', 'type': 'remote'})
|
||||
context = self.create(participates=False)
|
||||
self.state.original_prompt['end']['inputs']['value_1'] = ['start', 4]
|
||||
with self.assertRaisesRegex(ValueError, 'missing'):
|
||||
await self.rt.run_image_list_job(context, include_values=True)
|
||||
self.assertFalse(self.rt._MASTER_JOBS)
|
||||
|
||||
async def test_values_failure_and_cancellation_return_no_partial_lists(self):
|
||||
self.local_error = RuntimeError('branch failed')
|
||||
with self.assertRaisesRegex(RuntimeError, 'branch failed'):
|
||||
await self.rt.run_image_list_job(self.create(), include_values=True)
|
||||
self.local_error = None
|
||||
self.local_started.clear()
|
||||
self.local_delay = 0.1
|
||||
task = asyncio.create_task(self.rt.run_image_list_job(self.create(), include_values=True))
|
||||
await self.local_started.wait()
|
||||
task.cancel()
|
||||
with self.assertRaises(self.rt.ImageListCancelledError):
|
||||
await task
|
||||
self.assertFalse(self.rt._MASTER_JOBS)
|
||||
|
||||
async def test_local_failure_surfaces_and_reclaims_staged_token(self):
|
||||
self.local_error = RuntimeError('local branch failed')
|
||||
with self.assertRaisesRegex(RuntimeError, 'local branch failed'):
|
||||
@@ -373,6 +484,8 @@ def real_framework_probe(comfy_root):
|
||||
rt.remove_master_job(context)
|
||||
|
||||
for behavior in ('normal', 'expand', 'error', 'cancel', 'interrupt', 'sync', 'already-interrupted'):
|
||||
# Independent lifecycle cases start cold; retention is tested separately.
|
||||
local._RETAINED_OUTPUTS = None
|
||||
mode[0] = behavior
|
||||
prompt['op']['class_type'] = 'Expanded' if behavior == 'expand' else 'Operation'
|
||||
loads.clear()
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Image-list participation uses the existing master orchestrator setting."""
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from test_serverless_integration import PKG, load_modules
|
||||
|
||||
@@ -62,11 +63,21 @@ class MasterSelectionTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.q.dispatch_worker_prompt.assert_not_awaited()
|
||||
self.assertEqual(self.ps.distributed_pending_jobs, {})
|
||||
|
||||
async def test_end_overrides_require_value_capability_before_master_queue(self):
|
||||
self.prompt['3']['inputs']['value_10'] = ['2', 2]
|
||||
await self.queue()
|
||||
self.assertIs(self.preflight.call_args.kwargs['require_value_results'], True)
|
||||
self.q.queue_prompt_payload.reset_mock()
|
||||
self.preflight.side_effect = RuntimeError('value_results unsupported')
|
||||
with self.assertRaisesRegex(RuntimeError, 'value_results'):
|
||||
await self.queue()
|
||||
self.q.queue_prompt_payload.assert_not_awaited()
|
||||
|
||||
async def test_orchestrator_setting_excludes_master_processing(self):
|
||||
self.config["settings"]["master_delegate_only"] = True
|
||||
await self.queue()
|
||||
self.assertIs(self.prepared_inputs()["image_list_master_participates"], False)
|
||||
self.assertTrue(self.preflight.call_args.kwargs["require_idle"])
|
||||
self.assertIs(self.preflight.call_args.kwargs["require_idle"], False)
|
||||
# Upstream source stays on master even in orchestrator-only mode.
|
||||
self.assertIn("1", self.q.queue_prompt_payload.call_args.args[0])
|
||||
|
||||
@@ -97,10 +108,154 @@ class MasterSelectionTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), [])
|
||||
self.assertIs(self.prepared_inputs()["image_list_master_participates"], True)
|
||||
|
||||
async def test_busy_workers_do_not_prevent_enabled_master(self):
|
||||
self.preflight.return_value = {}
|
||||
def use_real_preflight(self, queues):
|
||||
"""Exercise capability/queue parsing with read-only fake HTTP responses."""
|
||||
from test_dispatch_selection import dispatch, _FakeSession, _valid_capability
|
||||
|
||||
responses = {(worker_id, "capabilities"): (200, _valid_capability())
|
||||
for worker_id in queues}
|
||||
responses.update({(worker_id, "queue"): (200, queue)
|
||||
for worker_id, queue in queues.items()})
|
||||
session = _FakeSession(responses)
|
||||
for target, value in (
|
||||
("get_client_session", AsyncMock(return_value=session)),
|
||||
("build_worker_url", lambda worker, path: f"http://{worker['id']}{path}"),
|
||||
):
|
||||
patched = patch.object(dispatch, target, value)
|
||||
patched.start()
|
||||
self.addCleanup(patched.stop)
|
||||
self.preflight = AsyncMock(wraps=dispatch.preflight_image_list_workers)
|
||||
sys.modules[PKG + ".api.orchestration.dispatch"].preflight_image_list_workers = self.preflight
|
||||
return responses
|
||||
|
||||
async def run_prepared_selection(self):
|
||||
"""Run real runtime selection up to the scheduler, without any item I/O."""
|
||||
rt = sys.modules[PKG + ".image_list.runtime"]
|
||||
inputs = self.prepared_inputs()
|
||||
image = object()
|
||||
context = rt.create_master_job(
|
||||
"2", "3", self.q.queue_prompt_payload.call_args.args[0],
|
||||
[{"image": image, "index": 0}],
|
||||
json.loads(inputs["image_list_enabled_worker_ids"]),
|
||||
inputs["image_list_master_callback"],
|
||||
master_participates=inputs["image_list_master_participates"],
|
||||
explicit_targets=True,
|
||||
)
|
||||
self.addCleanup(rt.remove_master_job, context)
|
||||
self.scheduler = AsyncMock(return_value=[{"image": image}])
|
||||
self.local_slot = Mock(return_value=types.SimpleNamespace(close=AsyncMock()))
|
||||
self.runtime_session = AsyncMock(return_value=object())
|
||||
config = sys.modules[PKG + ".utils.config"]
|
||||
network = sys.modules[PKG + ".utils.network"]
|
||||
with patch.object(config, "load_config", lambda: self.config), \
|
||||
patch.object(config, "get_worker_timeout_seconds", lambda: 60, create=True), \
|
||||
patch.object(config, "get_image_list_job_timeout_seconds", lambda: 0, create=True), \
|
||||
patch.object(network, "get_client_session", self.runtime_session, create=True), \
|
||||
patch.object(network, "build_worker_url", Mock(), create=True), \
|
||||
patch.object(rt, "dynamic_schedule", self.scheduler), \
|
||||
patch.dict(sys.modules, {PKG + ".image_list.local_executor": types.SimpleNamespace(
|
||||
LocalExecutionSlot=self.local_slot)}):
|
||||
await rt.run_image_list_job(context)
|
||||
self.assertIsNone(rt.get_master_job(context))
|
||||
return self.scheduler.call_args.args[1]
|
||||
|
||||
async def test_busy_worker_at_enqueue_is_available_to_scheduler_when_idle_at_start(self):
|
||||
responses = self.use_real_preflight({
|
||||
"worker": {"queue_running": [[1, "prompt-a"]], "queue_pending": []},
|
||||
})
|
||||
result = await self.queue()
|
||||
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), ["worker"])
|
||||
self.assertEqual(result[2], 1) # Candidates, not a promise of execution-time availability.
|
||||
self.q.dispatch_worker_prompt.assert_not_awaited()
|
||||
|
||||
responses[("worker", "queue")] = (200, {"queue_running": [], "queue_pending": []})
|
||||
# A newly enabled worker is not implicitly added to this queued selection.
|
||||
self.config["workers"].append({"id": "later", "enabled": True})
|
||||
slots = await self.run_prepared_selection()
|
||||
self.assertEqual([slot for slot in slots if isinstance(slot, str)], ["worker"])
|
||||
self.assertEqual(len(slots), 2)
|
||||
self.assertEqual(self.preflight.await_count, 2)
|
||||
self.assertEqual(self.preflight.await_args_list[0].args, self.preflight.await_args_list[1].args)
|
||||
|
||||
async def test_worker_still_busy_at_start_is_excluded_from_scheduler(self):
|
||||
self.use_real_preflight({
|
||||
"worker": {"queue_running": [], "queue_pending": [[1, "prompt-a"]]},
|
||||
})
|
||||
await self.queue()
|
||||
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), [])
|
||||
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), ["worker"])
|
||||
slots = await self.run_prepared_selection()
|
||||
self.assertEqual(len(slots), 1)
|
||||
self.assertNotIn("worker", slots)
|
||||
self.assertEqual(self.preflight.await_count, 2)
|
||||
self.runtime_session.assert_not_awaited()
|
||||
|
||||
async def test_explicit_empty_selection_stays_master_only_at_start(self):
|
||||
self.use_real_preflight({"worker": {"queue_running": [], "queue_pending": []}})
|
||||
await self.queue(enabled_worker_ids=[])
|
||||
slots = await self.run_prepared_selection()
|
||||
self.assertEqual(len(slots), 1)
|
||||
self.assertNotIn("worker", slots)
|
||||
self.preflight.assert_not_awaited()
|
||||
self.runtime_session.assert_not_awaited()
|
||||
|
||||
async def test_busy_incompatible_worker_fails_before_master_queue(self):
|
||||
responses = self.use_real_preflight({
|
||||
"worker": {"queue_running": [[1, "prompt-a"]], "queue_pending": []},
|
||||
})
|
||||
capability = responses[("worker", "capabilities")][1]
|
||||
for field, invalid, message in (
|
||||
("protocol_version", 1, "incompatible"),
|
||||
("execution_targets", False, "execution_targets"),
|
||||
("value_results", False, "value_results"),
|
||||
):
|
||||
with self.subTest(field=field), patch.dict(capability, {field: invalid}):
|
||||
if field == "value_results":
|
||||
self.prompt["3"]["inputs"]["value_10"] = ["2", 2]
|
||||
with self.assertRaisesRegex(RuntimeError, message):
|
||||
await self.queue()
|
||||
self.q.queue_prompt_payload.assert_not_awaited()
|
||||
self.q.dispatch_worker_prompt.assert_not_awaited()
|
||||
|
||||
async def test_orchestrator_with_all_workers_busy_queues_for_idle_start(self):
|
||||
responses = self.use_real_preflight({
|
||||
"worker": {"queue_running": [[1, "prompt-a"]], "queue_pending": []},
|
||||
})
|
||||
result = await self.queue(delegate_master=True)
|
||||
self.q.queue_prompt_payload.assert_awaited_once()
|
||||
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), ["worker"])
|
||||
self.assertIs(self.prepared_inputs()["image_list_master_participates"], False)
|
||||
self.assertEqual(result[2], 1)
|
||||
self.q.dispatch_worker_prompt.assert_not_awaited()
|
||||
|
||||
responses[("worker", "queue")] = (200, {"queue_running": [], "queue_pending": []})
|
||||
self.assertEqual(await self.run_prepared_selection(), ["worker"])
|
||||
self.local_slot.assert_not_called()
|
||||
self.assertEqual(self.preflight.await_count, 2)
|
||||
self.assertIs(self.preflight.await_args_list[0].kwargs["require_idle"], False)
|
||||
self.assertIs(self.preflight.await_args_list[1].kwargs["require_idle"], True)
|
||||
|
||||
async def test_orchestrator_rechecks_availability_without_master_fallback(self):
|
||||
responses = self.use_real_preflight({"worker": {"queue_running": [], "queue_pending": []}})
|
||||
await self.queue(delegate_master=True)
|
||||
responses[("worker", "queue")] = (200, {"queue_running": [[1, "other"]], "queue_pending": []})
|
||||
with self.assertRaisesRegex(RuntimeError, "No selected feature-compatible direct worker is idle"):
|
||||
await self.run_prepared_selection()
|
||||
self.assertEqual(self.preflight.await_count, 2)
|
||||
self.scheduler.assert_not_awaited()
|
||||
self.local_slot.assert_not_called()
|
||||
|
||||
async def test_orchestrator_retains_busy_candidates_when_another_worker_is_idle(self):
|
||||
self.config["workers"].append({"id": "idle", "enabled": True})
|
||||
responses = self.use_real_preflight({
|
||||
"worker": {"queue_running": [[1, "prompt-a"]], "queue_pending": []},
|
||||
"idle": {"queue_running": [], "queue_pending": []},
|
||||
})
|
||||
await self.queue(delegate_master=True)
|
||||
self.assertEqual(json.loads(self.prepared_inputs()["image_list_enabled_worker_ids"]), ["worker", "idle"])
|
||||
responses[("worker", "queue")] = (200, {"queue_running": [], "queue_pending": []})
|
||||
responses[("idle", "queue")] = (200, {"queue_running": [[1, "other"]], "queue_pending": []})
|
||||
self.assertEqual(await self.run_prepared_selection(), ["worker"])
|
||||
self.local_slot.assert_not_called()
|
||||
|
||||
async def test_orchestrator_without_workers_fails_before_queue(self):
|
||||
self.config["settings"]["master_delegate_only"] = True
|
||||
|
||||
@@ -0,0 +1,536 @@
|
||||
"""Stable content signatures with fresh, expiring delivery capabilities."""
|
||||
import asyncio
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from image_list import runtime as rt
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def item():
|
||||
return rt.normalize_image_list_inputs(
|
||||
torch.arange(18, dtype=torch.float64).reshape(1, 2, 3, 3),
|
||||
torch.zeros(1, 2, 3, dtype=torch.bfloat16),
|
||||
{'value_1': 'original', 'value_2': True, 'value_3': 7, 'value_4': 0.25,
|
||||
'value_10': torch.ones(2, 3, dtype=torch.float16)})[0]
|
||||
|
||||
|
||||
def graph():
|
||||
return {
|
||||
'source': {'class_type': 'Source', 'inputs': {}},
|
||||
'start': {'class_type': rt.START_CLASS, 'inputs': {
|
||||
'images': ['source', 0], 'image_list_execution_id': 'transient',
|
||||
'image_list_workflow_meta': {'job': 'metadata'},
|
||||
'image_list_enabled_worker_ids': '["worker"]',
|
||||
'image_list_master_callback': 'http://old-master',
|
||||
'values.value_1': 'inherited', 'worker_context_id': 'old-context'}},
|
||||
'op': {'class_type': 'Operation', 'inputs': {'image': ['start', 0]}},
|
||||
'end': {'class_type': rt.END_CLASS, 'inputs': {
|
||||
'images': ['op', 0], 'context': ['start', 3], 'value_3': ['start', 2]}},
|
||||
}
|
||||
|
||||
|
||||
def worker_prompt(token, context, data, *, local=True):
|
||||
prompt = graph()
|
||||
result = rt.build_worker_prompt(prompt, rt.validate_image_list_regions(prompt)[0],
|
||||
token, context, data['index'], 'http://master',
|
||||
item_identity=rt.item_content_identity(data))
|
||||
if local:
|
||||
result['start']['inputs']['image_list_local_execution'] = True
|
||||
result['end']['inputs']['image_list_local_execution'] = True
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_state():
|
||||
yield
|
||||
rt._WORKER_ITEMS.clear()
|
||||
rt._MASTER_JOBS.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def boundaries(monkeypatch):
|
||||
"""Load only the boundary module; restore every replaced module on exit."""
|
||||
name = '_native_cache_identity_tests'
|
||||
for suffix in ('', '.nodes', '.utils', '.image_list', '.api'):
|
||||
module = types.ModuleType(name + suffix)
|
||||
module.__path__ = [str(ROOT / suffix.lstrip('.').replace('.', '/'))]
|
||||
monkeypatch.setitem(sys.modules, name + suffix, module)
|
||||
monkeypatch.setitem(sys.modules, name + '.image_list.runtime', rt)
|
||||
io = types.SimpleNamespace(ComfyNode=type('ComfyNode', (), {}),
|
||||
NodeOutput=lambda *args: args)
|
||||
for cls in ('Image', 'Mask', 'String', 'Int', 'Float', 'Boolean'):
|
||||
setattr(io, cls, object())
|
||||
monkeypatch.setitem(sys.modules, 'comfy_api', types.ModuleType('comfy_api'))
|
||||
monkeypatch.setitem(sys.modules, 'comfy_api.v0_0_2', types.SimpleNamespace(io=io))
|
||||
monkeypatch.setitem(sys.modules, name + '.utils.async_helpers',
|
||||
types.SimpleNamespace(run_async_in_server_loop=lambda coro, **kw: asyncio.run(coro)))
|
||||
spec = importlib.util.spec_from_file_location(name + '.nodes.image_list', ROOT / 'nodes/image_list.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def start(boundaries, prompt):
|
||||
boundaries.DistributedImageListStart.hidden = types.SimpleNamespace(prompt=prompt, unique_id='start')
|
||||
return boundaries.DistributedImageListStart.execute(**prompt['start']['inputs'])
|
||||
|
||||
|
||||
def end(boundaries, prompt, cached):
|
||||
boundaries.DistributedImageListEnd.hidden = types.SimpleNamespace(prompt=prompt, unique_id='end')
|
||||
inputs = {k: v for k, v in prompt['end']['inputs'].items()
|
||||
if k not in ('images', 'context', 'value_3')}
|
||||
return boundaries.DistributedImageListEnd.execute(
|
||||
images=cached[0], context=cached[3], value_3=cached[2], **inputs)
|
||||
|
||||
|
||||
def stage(data):
|
||||
token = rt.create_worker_task_token()
|
||||
rt.store_worker_item(token, rt.encode_item(data))
|
||||
return token
|
||||
|
||||
|
||||
def master(data, token):
|
||||
context = rt.create_master_job('start', 'end', graph(), [data], ['worker'], '')
|
||||
rt.register_assignment(context, data['index'], 'worker', token)
|
||||
return context
|
||||
|
||||
|
||||
def test_identity_is_exact_typed_order_independent_and_not_transport_bounded(monkeypatch):
|
||||
original = item()
|
||||
identity = rt.item_content_identity(original)
|
||||
assert rt.is_valid_item_identity(identity)
|
||||
reordered = copy.deepcopy(original)
|
||||
reordered['values'] = dict(reversed(list(reordered['values'].items())))
|
||||
assert rt.item_content_identity(reordered) == identity
|
||||
strided = copy.deepcopy(original)
|
||||
strided['image'] = original['image'].transpose(1, 2).contiguous().transpose(1, 2)
|
||||
assert not strided['image'].is_contiguous()
|
||||
assert rt.item_content_identity(strided) == identity
|
||||
monkeypatch.setattr(rt, 'MAX_PAYLOAD_BYTES', 1)
|
||||
with pytest.raises(ValueError, match='transport limit'):
|
||||
rt.encode_item(original)
|
||||
assert rt.item_content_identity(original) == identity
|
||||
scalar_identities = set()
|
||||
for value in (True, 1, 1.0, '1', 0.0, -0.0):
|
||||
changed = copy.deepcopy(original)
|
||||
changed['values']['value_2'] = value
|
||||
scalar_identities.add(rt.item_content_identity(changed))
|
||||
assert len(scalar_identities) == 6
|
||||
|
||||
|
||||
@pytest.mark.parametrize('change', ['image', 'mask', 'value', 'index', 'dtype', 'shape', 'slot', 'missing-mask'])
|
||||
def test_every_item_component_invalidates_identity(change):
|
||||
original, changed = item(), item()
|
||||
if change in ('image', 'mask'):
|
||||
changed[change].reshape(-1)[0] = 1
|
||||
elif change == 'value':
|
||||
changed['values']['value_10'][0, 0] = 2
|
||||
elif change == 'index':
|
||||
changed['index'] = 1
|
||||
elif change == 'dtype':
|
||||
changed['image'] = changed['image'].float()
|
||||
elif change == 'shape':
|
||||
changed['values']['value_10'] = changed['values']['value_10'].reshape(3, 2)
|
||||
elif change == 'slot':
|
||||
changed['values']['value_9'] = changed['values'].pop('value_10')
|
||||
else:
|
||||
changed['mask'] = None
|
||||
assert rt.item_content_identity(original) != rt.item_content_identity(changed)
|
||||
|
||||
|
||||
def test_float64_low_bits_and_bfloat16_bytes_are_not_quantized():
|
||||
data, changed = item(), item()
|
||||
changed['image'].reshape(-1)[0] = torch.nextafter(torch.tensor(0., dtype=torch.float64),
|
||||
torch.tensor(1., dtype=torch.float64))
|
||||
assert rt.item_content_identity(data) != rt.item_content_identity(changed)
|
||||
changed = item()
|
||||
changed['mask'].view(torch.int16).reshape(-1)[0] = 1
|
||||
assert rt.item_content_identity(data) != rt.item_content_identity(changed)
|
||||
|
||||
|
||||
def test_stable_start_replaced_wholesale_and_legacy_graph_preserved():
|
||||
prompt = graph()
|
||||
frozen = copy.deepcopy(prompt)
|
||||
region = rt.validate_image_list_regions(prompt)[0]
|
||||
legacy = rt.build_worker_prompt(prompt, region, 'a', 'ctx-a', 0, 'http://a')
|
||||
assert legacy['start']['inputs']['worker_item_token'] == 'a'
|
||||
assert legacy['start']['inputs']['image_list_execution_id'] == 'transient'
|
||||
identity = rt.item_content_identity(item())
|
||||
a = rt.build_worker_prompt(prompt, region, 'a', 'ctx-a', 0, 'http://a', item_identity=identity)
|
||||
b = rt.build_worker_prompt(prompt, region, 'b', 'ctx-b', 0, 'http://b', item_identity=identity)
|
||||
assert a['start']['inputs'] == b['start']['inputs'] == {'worker_item_identity': identity}
|
||||
assert a['end']['inputs']['worker_context_id'] == 'ctx-a'
|
||||
assert b['end']['inputs']['worker_result_token'] == 'b'
|
||||
assert b['end']['inputs']['worker_item_identity'] == identity
|
||||
assert b['end']['inputs']['value_3'] == ['start', 2]
|
||||
assert prompt == frozen
|
||||
|
||||
|
||||
@pytest.mark.parametrize('bad', ['', 'sha256:' + 'z' * 64, 'a' * 64, False, ['bad']])
|
||||
def test_malformed_identity_rejected(bad):
|
||||
assert not rt.is_valid_item_identity(bad)
|
||||
prompt = graph()
|
||||
with pytest.raises((ValueError, TypeError), match='identity'):
|
||||
rt.build_worker_prompt(prompt, rt.validate_image_list_regions(prompt)[0],
|
||||
'token', 'context', 0, '', item_identity=bad)
|
||||
|
||||
|
||||
def test_cache_miss_then_hit_delivers_current_context_token_and_values(boundaries):
|
||||
data = item()
|
||||
a = stage(data)
|
||||
context_a = master(data, a)
|
||||
prompt_a = worker_prompt(a, context_a, data)
|
||||
cached = start(boundaries, prompt_a)
|
||||
assert cached[3] == [rt.item_content_identity(data)]
|
||||
assert 'image' not in rt._WORKER_ITEMS[a][1] # only a small, expiring receipt remains
|
||||
end(boundaries, prompt_a, cached)
|
||||
assert a not in rt._WORKER_ITEMS
|
||||
assert rt.get_master_job(context_a).results[0]['values'] == {'value_3': 0}
|
||||
b = stage(data)
|
||||
context_b = master(data, b)
|
||||
prompt_b = worker_prompt(b, context_b, data)
|
||||
assert prompt_a['start'] == prompt_b['start']
|
||||
# A native hit does NOT execute Start; End must consume this current upload.
|
||||
end(boundaries, prompt_b, cached)
|
||||
assert b not in rt._WORKER_ITEMS
|
||||
state = rt.get_master_job(context_b)
|
||||
assert state.expected_tokens[0] == b
|
||||
assert torch.equal(state.results[0]['image'], data['image'])
|
||||
assert rt._collect_output_lists([state.results[0]], state=state, include_values=True)[3][0]['value_3'] == 0
|
||||
with pytest.raises(RuntimeError, match='missing|expired|consumed'):
|
||||
end(boundaries, prompt_b, cached)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('miss', [False, True])
|
||||
def test_expired_item_cannot_deliver_even_with_cached_start(boundaries, miss):
|
||||
data = item()
|
||||
token = stage(data)
|
||||
context = master(data, token)
|
||||
prompt = worker_prompt(token, context, data)
|
||||
cached = start(boundaries, prompt) if miss else ([data['image']], [data['mask']], [0], [rt.item_content_identity(data)])
|
||||
_, staged = rt._WORKER_ITEMS[token]
|
||||
rt._WORKER_ITEMS[token] = (0, staged)
|
||||
with pytest.raises(RuntimeError, match='missing|expired|consumed'):
|
||||
end(boundaries, prompt, cached)
|
||||
assert not rt.get_master_job(context).received_indexes
|
||||
assert token not in rt._WORKER_ITEMS
|
||||
|
||||
|
||||
def test_identity_mismatch_on_start_and_on_cache_hit_fails_closed(boundaries):
|
||||
data, different = item(), item()
|
||||
different['image'][0, 0, 0, 0] = 3
|
||||
for hit in (False, True):
|
||||
token = stage(different)
|
||||
context = master(different, token)
|
||||
prompt = worker_prompt(token, context, data)
|
||||
cached = ([data['image']], [data['mask']], [0], [rt.item_content_identity(data)])
|
||||
with pytest.raises((ValueError, RuntimeError), match='identity'):
|
||||
end(boundaries, prompt, cached) if hit else start(boundaries, prompt)
|
||||
assert not rt.get_master_job(context).received_indexes
|
||||
rt.delete_worker_item(token)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('change', ['ambiguous', 'wrong-link', 'wrong-start-id', 'end-identity', 'end-index'])
|
||||
def test_start_requires_exact_paired_end_binding(boundaries, change):
|
||||
data = item()
|
||||
token = stage(data)
|
||||
prompt = worker_prompt(token, 'context', data)
|
||||
if change == 'ambiguous':
|
||||
prompt['second-end'] = copy.deepcopy(prompt['end'])
|
||||
elif change == 'wrong-link':
|
||||
prompt['end']['inputs']['context'] = ['start', 2]
|
||||
elif change == 'wrong-start-id':
|
||||
prompt['renamed'] = prompt.pop('start')
|
||||
elif change == 'end-identity':
|
||||
prompt['end']['inputs']['worker_item_identity'] = 'sha256:' + '0' * 64
|
||||
else:
|
||||
prompt['end']['inputs']['worker_item_index'] = 1
|
||||
boundaries.DistributedImageListStart.hidden = types.SimpleNamespace(prompt=prompt, unique_id='start')
|
||||
with pytest.raises((RuntimeError, ValueError), match='paired|identity|index'):
|
||||
boundaries.DistributedImageListStart.execute(worker_item_identity=rt.item_content_identity(data),
|
||||
image_list_local_execution=True)
|
||||
|
||||
|
||||
def test_end_checks_stable_context_before_delivering(boundaries):
|
||||
data = item()
|
||||
token = stage(data)
|
||||
context = master(data, token)
|
||||
prompt = worker_prompt(token, context, data)
|
||||
cached = list(start(boundaries, prompt))
|
||||
cached[3] = ['sha256:' + '0' * 64]
|
||||
with pytest.raises((RuntimeError, ValueError), match='identity|context'):
|
||||
end(boundaries, prompt, cached)
|
||||
assert not rt.get_master_job(context).received_indexes
|
||||
|
||||
|
||||
def test_legacy_worker_path_still_consumes_and_delivers_job_context(boundaries):
|
||||
data = item()
|
||||
token = stage(data)
|
||||
context = master(data, token)
|
||||
prompt = graph()
|
||||
prompt = rt.build_worker_prompt(prompt, rt.validate_image_list_regions(prompt)[0], token, context, 0, '')
|
||||
prompt['start']['inputs']['image_list_local_execution'] = True
|
||||
prompt['end']['inputs']['image_list_local_execution'] = True
|
||||
cached = start(boundaries, prompt)
|
||||
assert cached[3] == [context]
|
||||
assert token not in rt._WORKER_ITEMS
|
||||
end(boundaries, prompt, cached)
|
||||
assert rt.get_master_job(context).received_indexes == {0}
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fmt', ['exact', 'png'])
|
||||
def test_remote_cache_hit_submits_fresh_callback_metadata(boundaries, monkeypatch, fmt):
|
||||
data = item()
|
||||
token = stage(data)
|
||||
prompt = worker_prompt(token, 'fresh-context', data, local=False)
|
||||
prompt['end']['inputs'].update(transfer_format=fmt, transfer_quality=73)
|
||||
submit = AsyncMock()
|
||||
monkeypatch.setattr(boundaries, 'submit_worker_result', submit)
|
||||
cached = ([data['image']], [data['mask']], [0], [rt.item_content_identity(data)])
|
||||
result = end(boundaries, prompt, cached)
|
||||
assert len(result) == 4
|
||||
assert submit.await_args.args[:4] == ('http://master', token, 'fresh-context', 0)
|
||||
assert submit.await_args.args[4]['values']['value_3'] == {'type': 'INT', 'value': 0}
|
||||
payload = submit.await_args.args[4]
|
||||
if fmt == 'png':
|
||||
assert payload['image']['descriptor']['format'] == fmt
|
||||
assert payload['image']['descriptor']['quality'] == 73
|
||||
else:
|
||||
assert 'encoding' not in payload['image']
|
||||
assert token not in rt._WORKER_ITEMS
|
||||
|
||||
|
||||
def test_item_route_returns_server_computed_identity(boundaries, monkeypatch):
|
||||
from aiohttp import web
|
||||
name = '_native_cache_identity_tests'
|
||||
server = types.SimpleNamespace(PromptServer=types.SimpleNamespace(instance=types.SimpleNamespace(routes=web.RouteTableDef())))
|
||||
monkeypatch.setitem(sys.modules, 'server', server)
|
||||
spec = importlib.util.spec_from_file_location(name + '.api.image_list_routes', ROOT / 'api/image_list_routes.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||
spec.loader.exec_module(module)
|
||||
data = item()
|
||||
token = rt.create_worker_task_token()
|
||||
monkeypatch.setattr(module, '_json_body', AsyncMock(return_value={
|
||||
'token': token, 'context_id': 'context', 'item': rt.encode_item(data),
|
||||
'item_identity': 'sha256:' + '0' * 64}))
|
||||
response = asyncio.run(module.image_list_item_endpoint(object()))
|
||||
assert response.status == 200
|
||||
assert json.loads(response.body)['item_identity'] == rt.item_content_identity(data)
|
||||
assert len(rt._WORKER_ITEMS[token]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize('responses', [
|
||||
('valid', 'valid'), ('valid', 'absent'), ('invalid', 'valid'),
|
||||
('bad-json', 'valid'), ('non-object', 'valid'),
|
||||
])
|
||||
def test_runtime_opts_in_per_upload_and_can_downgrade(monkeypatch, responses):
|
||||
name = '_native_cache_identity_runtime'
|
||||
for suffix in ('', '.image_list', '.api', '.api.orchestration', '.utils'):
|
||||
module = types.ModuleType(name + suffix)
|
||||
module.__path__ = [str(ROOT / suffix.lstrip('.').replace('.', '/'))]
|
||||
monkeypatch.setitem(sys.modules, name + suffix, module)
|
||||
spec = importlib.util.spec_from_file_location(name + '.image_list.runtime', ROOT / 'image_list/runtime.py')
|
||||
runtime = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, runtime)
|
||||
spec.loader.exec_module(runtime)
|
||||
workers = [{'id': 'worker', 'type': 'local'}]
|
||||
dispatched, uploads = [], []
|
||||
preflight = AsyncMock(return_value={'worker': True})
|
||||
sync_media = AsyncMock()
|
||||
|
||||
class Response:
|
||||
status = 200
|
||||
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args):
|
||||
pass
|
||||
|
||||
async def json(self):
|
||||
if self.data == 'bad-json':
|
||||
raise ValueError('old response is not JSON')
|
||||
return self.data
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
class Session:
|
||||
def post(self, url, *, data, **kw):
|
||||
assert url.endswith('/distributed/image-list/item')
|
||||
body = json.loads(data)
|
||||
identity = runtime.store_worker_item(body['token'], body['item'])
|
||||
mode = responses[len(uploads)]
|
||||
uploads.append(body)
|
||||
response = {'valid': {'item_identity': identity}, 'absent': {'status': 'stored'},
|
||||
'invalid': {'item_identity': 'bad'}, 'bad-json': 'bad-json', 'non-object': []}[mode]
|
||||
return Response(response)
|
||||
|
||||
def get(self, url, **kw):
|
||||
pid = url.rsplit('/', 1)[-1]
|
||||
return Response({pid: {'status': {'completed': True, 'status_str': 'success'}}})
|
||||
|
||||
def delete(self, url, **kw):
|
||||
runtime.delete_worker_item(url.rsplit('/', 1)[-1])
|
||||
return Response({})
|
||||
|
||||
async def dispatch(worker, prompt, meta, **kwargs):
|
||||
dispatched.append(copy.deepcopy(prompt))
|
||||
delivery = prompt['end']['inputs']
|
||||
token, index = delivery['worker_result_token'], delivery['worker_item_index']
|
||||
identity = prompt['start']['inputs'].get('worker_item_identity')
|
||||
options = {'item_identity': identity, 'item_index': index} if identity else {}
|
||||
data = runtime.consume_worker_item(token, **options)
|
||||
if identity:
|
||||
runtime.finish_worker_item(token, identity, index)
|
||||
runtime.set_master_result(context, index, token, {
|
||||
'image': data['image'], 'text': str(index), 'values': {'value_3': index}})
|
||||
assert kwargs['partial_execution_targets'] == ['end']
|
||||
return f'worker-{index}'
|
||||
|
||||
dependencies = {
|
||||
'.api.orchestration.dispatch': types.SimpleNamespace(
|
||||
preflight_image_list_workers=preflight, dispatch_worker_prompt=dispatch),
|
||||
'.api.orchestration.prompt_transform': types.SimpleNamespace(
|
||||
_resolve_direct_workers_for_image_list=lambda *args: workers),
|
||||
'.api.orchestration.media_sync': types.SimpleNamespace(sync_worker_media=sync_media),
|
||||
'.utils.config': types.SimpleNamespace(load_config=lambda: {'settings': {}},
|
||||
get_worker_timeout_seconds=lambda: 60),
|
||||
'.utils.network': types.SimpleNamespace(build_master_callback_url=lambda *a, **kw: 'http://master',
|
||||
build_worker_url=lambda worker, path: 'http://worker' + path,
|
||||
get_client_session=AsyncMock(return_value=Session())),
|
||||
'.utils.logging': types.SimpleNamespace(log=lambda *args: None),
|
||||
}
|
||||
for suffix, module in dependencies.items():
|
||||
monkeypatch.setitem(sys.modules, name + suffix, module)
|
||||
monkeypatch.setitem(sys.modules, 'aiohttp', types.SimpleNamespace(
|
||||
ClientTimeout=lambda **kwargs: kwargs, ClientError=ConnectionError))
|
||||
comfy = types.ModuleType('comfy')
|
||||
comfy.model_management = types.SimpleNamespace(processing_interrupted=lambda: False)
|
||||
monkeypatch.setitem(sys.modules, 'comfy', comfy)
|
||||
monkeypatch.setitem(sys.modules, 'comfy.model_management', comfy.model_management)
|
||||
items = [item(), item()]
|
||||
items[1]['index'] = 1
|
||||
context = runtime.create_master_job('start', 'end', graph(), items, ['worker'], '')
|
||||
state = runtime.get_master_job(context)
|
||||
try:
|
||||
result = asyncio.run(runtime._run_image_list_job(state, include_values=True))
|
||||
assert result[2] == ['0', '1']
|
||||
assert [row['value_3'] for row in result[3]] == [0, 1]
|
||||
assert len(uploads) == len(dispatched) == 2
|
||||
assert sync_media.await_count == 1
|
||||
for index, prompt in enumerate(dispatched):
|
||||
start_inputs = prompt['start']['inputs']
|
||||
if responses[index] == 'valid':
|
||||
assert start_inputs == {'worker_item_identity': runtime.item_content_identity(items[index])}
|
||||
assert prompt['end']['inputs']['worker_context_id'] == context
|
||||
else:
|
||||
assert 'worker_item_identity' not in start_inputs
|
||||
assert start_inputs['worker_item_token'] == uploads[index]['token']
|
||||
assert start_inputs['worker_context_id'] == context
|
||||
assert 'worker_item_identity' not in prompt['end']['inputs']
|
||||
assert prompt['end']['inputs']['worker_result_token'] == uploads[index]['token']
|
||||
assert not runtime._WORKER_ITEMS
|
||||
finally:
|
||||
runtime.remove_master_job(context)
|
||||
runtime._WORKER_ITEMS.clear()
|
||||
|
||||
|
||||
def test_runtime_local_path_hashes_without_transport(monkeypatch):
|
||||
name = '_native_cache_identity_local_runtime'
|
||||
for suffix in ('', '.image_list', '.api', '.api.orchestration', '.utils'):
|
||||
module = types.ModuleType(name + suffix)
|
||||
module.__path__ = [str(ROOT / suffix.lstrip('.').replace('.', '/'))]
|
||||
monkeypatch.setitem(sys.modules, name + suffix, module)
|
||||
spec = importlib.util.spec_from_file_location(name + '.image_list.runtime', ROOT / 'image_list/runtime.py')
|
||||
runtime = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, runtime)
|
||||
spec.loader.exec_module(runtime)
|
||||
dispatched = []
|
||||
|
||||
class Slot:
|
||||
def __init__(self, **kw):
|
||||
pass
|
||||
|
||||
def submit(self, prompt, prompt_id, end_id, workflow_meta):
|
||||
dispatched.append(prompt)
|
||||
delivery = prompt[end_id]['inputs']
|
||||
identity = prompt['start']['inputs']['worker_item_identity']
|
||||
data = runtime.consume_worker_item(delivery['worker_result_token'], item_identity=identity,
|
||||
item_index=delivery['worker_item_index'])
|
||||
runtime.finish_worker_item(delivery['worker_result_token'], identity, data['index'])
|
||||
runtime.set_master_result(delivery['worker_context_id'], data['index'], delivery['worker_result_token'],
|
||||
{'image': data['image'], 'values': {'value_3': data['index']}})
|
||||
future = asyncio.get_running_loop().create_future()
|
||||
future.set_result(None)
|
||||
return future
|
||||
|
||||
def request_stop(self):
|
||||
pass
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('local path attempted HTTP/transport')
|
||||
|
||||
dependencies = {
|
||||
'.image_list.local_executor': types.SimpleNamespace(LocalExecutionSlot=Slot),
|
||||
'.api.orchestration.dispatch': types.SimpleNamespace(
|
||||
preflight_image_list_workers=forbidden, dispatch_worker_prompt=forbidden),
|
||||
'.api.orchestration.prompt_transform': types.SimpleNamespace(
|
||||
_resolve_direct_workers_for_image_list=lambda *args: []),
|
||||
'.utils.config': types.SimpleNamespace(load_config=lambda: {'settings': {}},
|
||||
get_worker_timeout_seconds=lambda: 60),
|
||||
'.utils.network': types.SimpleNamespace(build_master_callback_url=forbidden,
|
||||
build_worker_url=forbidden, get_client_session=forbidden),
|
||||
'.utils.logging': types.SimpleNamespace(log=lambda *args: None),
|
||||
}
|
||||
for suffix, module in dependencies.items():
|
||||
monkeypatch.setitem(sys.modules, name + suffix, module)
|
||||
monkeypatch.setitem(sys.modules, 'aiohttp', types.SimpleNamespace(
|
||||
ClientTimeout=lambda **kwargs: kwargs, ClientError=ConnectionError))
|
||||
comfy = types.ModuleType('comfy')
|
||||
comfy.model_management = types.SimpleNamespace(processing_interrupted=lambda: False)
|
||||
monkeypatch.setitem(sys.modules, 'comfy', comfy)
|
||||
monkeypatch.setitem(sys.modules, 'comfy.model_management', comfy.model_management)
|
||||
monkeypatch.setattr(runtime, 'MAX_PAYLOAD_BYTES', 1)
|
||||
monkeypatch.setattr(runtime, 'encode_item', forbidden)
|
||||
data = item()
|
||||
context = runtime.create_master_job('start', 'end', graph(), [data], [], '', master_participates=True)
|
||||
try:
|
||||
result = asyncio.run(runtime._run_image_list_job(runtime.get_master_job(context), include_values=True))
|
||||
assert dispatched[0]['start']['inputs'] == {'worker_item_identity': runtime.item_content_identity(data),
|
||||
'image_list_local_execution': True}
|
||||
assert result[3][0]['value_3'] == 0
|
||||
assert not runtime._WORKER_ITEMS
|
||||
finally:
|
||||
runtime.remove_master_job(context)
|
||||
runtime._WORKER_ITEMS.clear()
|
||||
|
||||
|
||||
def test_real_native_signatures_and_cached_delivery():
|
||||
core = Path(os.environ.get('COMFYUI_SOURCE_ROOT', str(Path(sys.executable).parents[2])))
|
||||
if not (core / 'comfy_execution/caching.py').exists():
|
||||
pytest.skip('real ComfyUI source not available beside this interpreter')
|
||||
result = subprocess.run([sys.executable, str(ROOT / 'tests/helpers/image_list_native_cache_identity.py'), str(core)],
|
||||
text=True, capture_output=True, timeout=90)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert 'NATIVE_CACHE_IDENTITY_OK' in result.stdout
|
||||
@@ -345,6 +345,7 @@ class LifecycleTests(unittest.IsolatedAsyncioTestCase):
|
||||
index = int(media["request_id"].rsplit(":", 1)[1])
|
||||
data = item(index)
|
||||
values = {"images": data["image"], "masks": data["mask"], "texts": str(index)}
|
||||
values.update(getattr(self, "override_values", {}))
|
||||
for n, binding in enumerate(media["outputs"]):
|
||||
descriptor, payload = codec.encode_payload(binding["kind"], values[binding["socket"]])
|
||||
aid = f"result:{n}"
|
||||
@@ -671,6 +672,50 @@ class LifecycleTests(unittest.IsolatedAsyncioTestCase):
|
||||
assert not self.server.distributed_pending_jobs
|
||||
self.api.submit.assert_not_awaited()
|
||||
|
||||
async def test_sparse_overrides_overlay_master_values_without_forwarding(self):
|
||||
from test_image_list_serverless_values import assert_exact
|
||||
self.override_values = {"value_2": 2**63 + 333, "value_10": False}
|
||||
self.state.original_prompt["ns/end"]["inputs"].update(
|
||||
value_2=["process", 2], value_10=["process", 3])
|
||||
slot, handle = await self.dispatch()
|
||||
await self.finish(handle)
|
||||
result = self.controller.poll({slot: handle})[2]
|
||||
assert set(result["values"]) == set(self.override_values)
|
||||
for name, expected in self.override_values.items():
|
||||
assert_exact(result["values"][name], expected)
|
||||
state = copy.copy(self.state)
|
||||
state.items = [self.state.items[0]]
|
||||
columns = runtime._collect_output_lists([result], state=state, include_values=True)
|
||||
assert columns[3][0] == {**state.items[0]["values"], **self.override_values}
|
||||
assert self.state.items[0]["values"]["value_2"] == 123
|
||||
|
||||
async def test_override_missing_extra_wrong_nonfinite_multiple_results_fail(self):
|
||||
self.override_values = {"value_2": 42}
|
||||
self.state.original_prompt["ns/end"]["inputs"]["value_2"] = ["process", 2]
|
||||
for index, bad in enumerate(("missing", "extra", "wrong", "nonfinite", "multiple", "none")):
|
||||
slot, handle = await self.dispatch(index)
|
||||
await self.finish(handle)
|
||||
queue = self.server.distributed_pending_jobs[handle["job_id"]]
|
||||
message = queue.get_nowait()
|
||||
media = message["media"]
|
||||
if bad == "missing":
|
||||
media.pop("value_2")
|
||||
elif bad == "extra":
|
||||
media["value_3"] = [1]
|
||||
else:
|
||||
media["value_2"] = {"wrong": [{}], "nonfinite": [float("nan")],
|
||||
"multiple": [1, 2], "none": [None]}[bad]
|
||||
queue.put_nowait(message)
|
||||
with pytest.raises((ValueError, TypeError, RuntimeError)):
|
||||
self.controller.poll({slot: handle})
|
||||
|
||||
async def test_no_override_results_preserve_original_return_shape(self):
|
||||
slot, handle = await self.dispatch()
|
||||
await self.finish(handle)
|
||||
result = self.controller.poll({slot: handle})[2]
|
||||
assert set(result) == {"image", "mask", "text"}
|
||||
assert not any(binding["kind"] == "VALUE" for binding in self.jobs["provider-0"]["media"]["outputs"])
|
||||
|
||||
async def test_duplicate_index_and_mismatched_handle_fail(self):
|
||||
slot, handle = await self.dispatch()
|
||||
with pytest.raises(ValueError, match="duplicate"):
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Sparse End overrides through the standalone serverless bridge, CPU only."""
|
||||
import asyncio
|
||||
import copy
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from test_image_list_serverless import adapter, codec, contract, handler, item, prepare, source
|
||||
|
||||
|
||||
@pytest.mark.skipif(not os.environ.get('COMFYUI_SOURCE_ROOT'), reason='set COMFYUI_SOURCE_ROOT for real CPU executor')
|
||||
def test_real_cpu_standalone_bridge_calculated_overrides(tmp_path):
|
||||
env = dict(os.environ, CUDA_VISIBLE_DEVICES='', HIP_VISIBLE_DEVICES='', ROCR_VISIBLE_DEVICES='',
|
||||
PYTHONDONTWRITEBYTECODE='1')
|
||||
env.pop('PYTHONPATH', None)
|
||||
helper = Path(__file__).parent / 'helpers' / 'image_list_serverless_values_check.py'
|
||||
result = subprocess.run([sys.executable, str(helper), os.environ['COMFYUI_SOURCE_ROOT'], str(tmp_path)],
|
||||
env=env, text=True, capture_output=True, timeout=90)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert 'IMAGE_LIST_SERVERLESS_VALUES_CPU_OK' in result.stdout
|
||||
|
||||
|
||||
def assert_exact(actual, expected):
|
||||
assert type(actual) is type(expected)
|
||||
if isinstance(expected, torch.Tensor):
|
||||
assert actual.dtype == expected.dtype and actual.shape == expected.shape
|
||||
assert torch.equal(actual.contiguous().view(torch.uint8), expected.contiguous().view(torch.uint8))
|
||||
elif isinstance(expected, float):
|
||||
assert struct.pack('!d', actual) == struct.pack('!d', expected)
|
||||
else:
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize('value', [False, True, 2**63 + 123, -(2**100), -0.0, 1.0000000000000002,
|
||||
'', ' \r\n雪 ', torch.ones(2, 2, 3, 4, dtype=torch.float64),
|
||||
torch.ones(2, 2, 3, dtype=torch.bfloat16)])
|
||||
def test_value_codec_exact_types(value):
|
||||
descriptor, payload = codec.encode_payload('VALUE', value)
|
||||
assert_exact(codec.decode_payload('VALUE', descriptor, payload), value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('value', [None, {}, [], (1,), object(), float('inf'), float('nan'),
|
||||
torch.ones(2), torch.ones(1, 2, 3, dtype=torch.int64),
|
||||
torch.full((1, 2, 3), float('nan'))])
|
||||
def test_value_codec_rejects_unsupported_or_nonfinite(value):
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
codec.encode_payload('VALUE', value)
|
||||
|
||||
|
||||
def test_sparse_exports_never_create_collector_or_forward_unchanged_values():
|
||||
async def check():
|
||||
prompt, value = source(), item()
|
||||
prompt['ns/end']['inputs'].update(value_2=['ns/start', 5], value_10=['process', 2])
|
||||
original = copy.deepcopy(prompt)
|
||||
job = await prepare(prompt, value)
|
||||
assert prompt == original
|
||||
bindings = {binding['socket']: binding for binding in job['manifest']}
|
||||
assert set(bindings) == {'images', 'masks', 'texts', 'value_2', 'value_10'}
|
||||
assert bindings['value_2']['kind'] == bindings['value_10']['kind'] == 'VALUE'
|
||||
assert all('Collector' not in node['class_type'] for node in job['workflow'].values())
|
||||
assert all(node['class_type'] not in (adapter.runtime.START_CLASS, adapter.runtime.END_CLASS)
|
||||
for node in job['workflow'].values())
|
||||
handler._validate_request({key: job[key] for key in ('workflow', 'images', 'media')})
|
||||
asyncio.run(check())
|
||||
|
||||
|
||||
@pytest.mark.parametrize('slot', ['value_0', 'value_11', 'value_01', 'values.value_1'])
|
||||
def test_unknown_end_override_slot_rejected(slot):
|
||||
prompt = source()
|
||||
prompt['ns/end']['inputs'][slot] = ['ns/start', 4]
|
||||
with pytest.raises(ValueError, match='slot'):
|
||||
asyncio.run(prepare(prompt))
|
||||
|
||||
|
||||
def test_value_codec_budgets_and_malformed_payload():
|
||||
with pytest.raises(ValueError, match='byte limit'):
|
||||
codec.encode_payload('VALUE', '雪', max_bytes=2)
|
||||
descriptor, payload = codec.encode_payload('VALUE', True)
|
||||
with pytest.raises(ValueError):
|
||||
codec.decode_payload('VALUE', dict(descriptor, type='INT'), payload)
|
||||
descriptor, payload = codec.encode_payload('VALUE', -0.0)
|
||||
with pytest.raises(ValueError):
|
||||
codec.decode_payload('VALUE', descriptor, struct.pack('<d', math.inf))
|
||||
with pytest.raises(ValueError):
|
||||
codec.decode_payload('VALUE', descriptor, payload + b'0')
|
||||
with pytest.raises(ValueError, match='byte limit'):
|
||||
codec.decode_payload('VALUE', descriptor, payload, max_bytes=7)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fmt', ['exact', 'legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1'])
|
||||
def test_end_image_policy_never_compresses_override_tensors(fmt):
|
||||
prompt = source()
|
||||
prompt['ns/end']['inputs'].update(value_10=['ns/start', 0], transfer_format=fmt, transfer_quality=73)
|
||||
job = asyncio.run(prepare(prompt))
|
||||
for binding in job['manifest']:
|
||||
controls = job['workflow'][binding['node_id']]['inputs']
|
||||
if binding['socket'] == 'images' and fmt != 'exact':
|
||||
assert controls['transfer_format'] == fmt and controls['transfer_quality'] == 73
|
||||
else:
|
||||
assert 'transfer_format' not in controls and 'transfer_quality' not in controls
|
||||
default = asyncio.run(prepare())
|
||||
assert all('transfer_format' not in n['inputs'] and 'transfer_quality' not in n['inputs']
|
||||
for n in default['workflow'].values())
|
||||
|
||||
|
||||
def value_response():
|
||||
binding = dict(binding_id='binding', collector_id='end', job_id='job', participant_id='pool',
|
||||
node_id='export', socket='value_10', kind='VALUE')
|
||||
descriptor, payload = codec.encode_payload('VALUE', 2**63 + 7)
|
||||
artifact = contract.artifact_from_payload('artifact', 'VALUE', descriptor, payload)
|
||||
return binding, dict(version=1, profile=contract.PROFILE, request_id='request',
|
||||
bindings=[dict(binding, count=1)], artifacts=[artifact],
|
||||
items=[dict(binding_id='binding', item_index=0, artifact_id='artifact')])
|
||||
|
||||
|
||||
@pytest.mark.parametrize('bad', ['missing', 'extra', 'wrong_kind', 'multiple', 'extra_artifact', 'nonfinite'])
|
||||
def test_override_response_rejected_atomically(bad):
|
||||
binding, response = value_response()
|
||||
if bad == 'missing':
|
||||
response['bindings'] = []
|
||||
elif bad == 'extra':
|
||||
response['bindings'].append(dict(binding, binding_id='other', socket='value_9', count=1))
|
||||
elif bad == 'wrong_kind':
|
||||
response['artifacts'][0]['kind'] = 'STRING'
|
||||
elif bad == 'multiple':
|
||||
response['bindings'][0]['count'] = 2
|
||||
elif bad == 'extra_artifact':
|
||||
response['artifacts'].append(dict(response['artifacts'][0], id='extra'))
|
||||
else:
|
||||
descriptor, _ = codec.encode_payload('VALUE', 1.0)
|
||||
response['artifacts'][0] = contract.artifact_from_payload(
|
||||
'artifact', 'VALUE', descriptor, struct.pack('<d', float('nan')))
|
||||
with pytest.raises(ValueError):
|
||||
asyncio.run(contract.decode_response(response, request_id='request', bindings=[binding]))
|
||||
|
||||
|
||||
def test_override_request_and_result_budgets():
|
||||
prompt = source()
|
||||
prompt['ns/end']['inputs']['value_10'] = '雪' * 100
|
||||
with patch.object(contract, 'MAX_ARTIFACT_BYTES', 128):
|
||||
with pytest.raises(ValueError, match='byte limit'):
|
||||
asyncio.run(prepare(prompt))
|
||||
binding, response = value_response()
|
||||
with pytest.raises(ValueError, match='total byte limit'):
|
||||
asyncio.run(contract.decode_response(response, request_id='request', bindings=[binding], max_total_bytes=1))
|
||||
|
||||
|
||||
def test_collector_compiler_keeps_only_its_public_sockets():
|
||||
from importlib import import_module
|
||||
compiler = import_module(adapter.__package__.rsplit('.', 1)[0] + '.api.orchestration.serverless_media_prompt')
|
||||
assert compiler.COLLECTOR_SOCKETS == ('images', 'audio')
|
||||
graph = {'source': {'class_type': 'Source', 'inputs': {}},
|
||||
'collector': {'class_type': 'DistributedCollector', 'inputs': {
|
||||
'images': ['source', 0], 'value_10': ['source', 1], 'masks': ['source', 2],
|
||||
'multi_job_id': 'job', 'worker_id': 'pool'}}}
|
||||
_, bindings = compiler.compile_media_prompt(graph, 'request')
|
||||
assert [binding['socket'] for binding in bindings] == ['images']
|
||||
@@ -147,6 +147,36 @@ def test_compressed_request_combined_budget(monkeypatch):
|
||||
rt.decode_result(encoded)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fmt", FORMATS)
|
||||
def test_return_formats_preserve_exact_sparse_override_values(fmt):
|
||||
from test_image_list_values import assert_values, values
|
||||
|
||||
source = result()
|
||||
source["values"] = values()
|
||||
source["values"]["value_10"] = source["values"].pop("value_6")
|
||||
encoded = rt.encode_result(source, transfer_format=fmt, transfer_quality=73)
|
||||
assert set(encoded["values"]) == set(source["values"])
|
||||
for name, value in source["values"].items():
|
||||
assert encoded["values"][name] == rt.encode_value(value, rt.value_kind(value, name))
|
||||
decoded = rt.decode_result(json.loads(rt.encode_request_payload(encoded)))
|
||||
assert_values(decoded["values"], source["values"])
|
||||
assert torch.equal(decoded["mask"], source["mask"])
|
||||
assert decoded["text"] == source["text"]
|
||||
|
||||
|
||||
def test_compressed_aggregate_budget_includes_override_values(monkeypatch):
|
||||
source = result()
|
||||
encoded_without_values = rt.encode_result(source, transfer_format="png")
|
||||
source["values"] = {"value_10": "x" * 500}
|
||||
encoded = rt.encode_result(source, transfer_format="png")
|
||||
monkeypatch.setattr(rt, "MAX_REQUEST_BYTES", len(rt.encode_request_payload(encoded_without_values)) + 100)
|
||||
with pytest.raises(ValueError, match="request"):
|
||||
rt.encode_result(source, transfer_format="png")
|
||||
with patch.object(image_codec, "decode_image", side_effect=AssertionError("oversize reached codec")):
|
||||
with pytest.raises(ValueError, match="request"):
|
||||
rt.decode_result(encoded)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dispatch_module(monkeypatch):
|
||||
name = "_image_list_transfer_tests"
|
||||
@@ -223,11 +253,15 @@ def test_capability_endpoint_advertises_shared_formats(monkeypatch):
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
response = asyncio.run(module.image_list_capabilities_endpoint(None))
|
||||
assert json.loads(response.text)["image_transfer_formats"] == list(FORMATS)
|
||||
caps = json.loads(response.text)
|
||||
assert caps["image_transfer_formats"] == list(FORMATS)
|
||||
assert all(caps[name] is True for name in ("supported", "execution_targets", "value_results"))
|
||||
assert "DistributedImageListGetValue" in caps["node_ids"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reject_call", [None, 1, 2, 3])
|
||||
def test_runtime_initial_and_each_nonexact_assignment_preflight(reject_call):
|
||||
@pytest.mark.parametrize("with_values", [False, True])
|
||||
def test_runtime_initial_and_each_nonexact_assignment_preflight(reject_call, with_values):
|
||||
from test_image_list_runtime_boundaries import RuntimeBoundaryTests
|
||||
|
||||
async def exercise():
|
||||
@@ -237,11 +271,22 @@ def test_runtime_initial_and_each_nonexact_assignment_preflight(reject_call):
|
||||
context = fixture.create(2)
|
||||
state = fixture.rt.get_master_job(context)
|
||||
state.original_prompt[state.end_id]["inputs"].update(transfer_format="png", transfer_quality=87)
|
||||
if with_values:
|
||||
state.original_prompt[state.end_id]["inputs"]["value_10"] = [state.start_id, 2]
|
||||
for item in state.items:
|
||||
item["values"] = {"value_1": "retained"}
|
||||
original_set = fixture.rt.set_master_result
|
||||
|
||||
def set_with_override(context, index, token, result):
|
||||
return original_set(context, index, token, {**result, "values": {"value_10": index}})
|
||||
|
||||
fixture.rt.set_master_result = set_with_override
|
||||
calls = []
|
||||
|
||||
async def preflight(workers, **kwargs):
|
||||
calls.append(kwargs)
|
||||
assert kwargs["required_transfer_formats"] == ("png",)
|
||||
assert kwargs.get("require_value_results", False) is with_values
|
||||
if len(calls) == reject_call:
|
||||
raise RuntimeError("upgrade worker; no work was submitted")
|
||||
return {"worker": True}
|
||||
@@ -249,13 +294,15 @@ def test_runtime_initial_and_each_nonexact_assignment_preflight(reject_call):
|
||||
sys.modules["_image_list_boundaries.api.orchestration.dispatch"].preflight_image_list_workers = preflight
|
||||
if reject_call:
|
||||
with pytest.raises(RuntimeError, match="upgrade"):
|
||||
await fixture.rt.run_image_list_job(context)
|
||||
await fixture.rt.run_image_list_job(context, include_values=with_values)
|
||||
assert len(fixture.uploads) == max(0, reject_call - 2)
|
||||
assert len(fixture.prompts) == max(0, reject_call - 2)
|
||||
else:
|
||||
await fixture.rt.run_image_list_job(context)
|
||||
result_lists = await fixture.rt.run_image_list_job(context, include_values=with_values)
|
||||
assert len(calls) == 3
|
||||
assert len(fixture.uploads) == 2
|
||||
if with_values:
|
||||
assert result_lists[3] == [{"value_1": "retained", "value_10": index} for index in range(2)]
|
||||
assert not fixture.rt._MASTER_JOBS
|
||||
finally:
|
||||
# Do not call doCleanups(): this borrowed IsolatedAsyncioTestCase has
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Context-carried values and sparse, per-item End overrides."""
|
||||
import copy
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from image_list import runtime as rt
|
||||
from image_list.execution_plan import prepare_image_list_execution
|
||||
|
||||
|
||||
def value_graph():
|
||||
# Use a minimal, closed region with an override-only ancestor.
|
||||
return {
|
||||
'source': {'class_type': 'Source', 'inputs': {}},
|
||||
'start': {'class_type': rt.START_CLASS, 'inputs': {'images': ['source', 0]}},
|
||||
'override': {'class_type': 'Transform', 'inputs': {'value': ['start', 4]}},
|
||||
'end': {'class_type': rt.END_CLASS, 'inputs': {
|
||||
'images': ['start', 0], 'context': ['start', 3], 'value_1': ['override', 0],
|
||||
'value_10': ['start', 5]}},
|
||||
}
|
||||
|
||||
|
||||
def values():
|
||||
return {'value_1': torch.arange(72, dtype=torch.float64).reshape(2, 3, 4, 3),
|
||||
'value_2': torch.ones(3, 2, 2, dtype=torch.bfloat16), 'value_3': 'name',
|
||||
'value_4': 42, 'value_5': 0.125, 'value_6': True}
|
||||
|
||||
|
||||
def assert_values(actual, expected):
|
||||
assert actual.keys() == expected.keys()
|
||||
for key, value in expected.items():
|
||||
assert type(actual[key]) is type(value)
|
||||
if isinstance(value, torch.Tensor):
|
||||
assert actual[key].dtype == value.dtype
|
||||
assert torch.equal(actual[key], value)
|
||||
else:
|
||||
assert actual[key] == value
|
||||
|
||||
|
||||
def test_normalize_and_roundtrip_all_override_kinds_keeps_batches_whole():
|
||||
expected = values()
|
||||
result = rt.normalize_worker_result(torch.zeros(1, 2, 2, 3),
|
||||
values={key: [value] for key, value in expected.items()})
|
||||
assert_values(result['values'], expected)
|
||||
wire = rt.encode_result(result)
|
||||
assert_values(rt.decode_result(wire)['values'], expected)
|
||||
assert wire['values']['value_1']['shape'][0] == 2
|
||||
assert wire['values']['value_2']['shape'][0] == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize('bad', [None, [], [None], [1, 2], [[1]], object(),
|
||||
float('nan'), float('inf'), torch.tensor([float('nan')]),
|
||||
torch.tensor([float('inf')]), torch.ones(1, dtype=torch.int32)])
|
||||
def test_invalid_override_rejected(bad):
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
rt.normalize_worker_result(torch.zeros(1, 2, 2, 3), values={'value_1': bad})
|
||||
|
||||
|
||||
def test_unknown_override_names_and_wire_types_rejected():
|
||||
for name in ('value_0', 'value_11', 'values.value_1'):
|
||||
with pytest.raises(ValueError, match='slot'):
|
||||
rt.normalize_worker_result(torch.zeros(1, 2, 2, 3), values={name: 1})
|
||||
base = rt.encode_result({'image': torch.zeros(1, 2, 2, 3)})
|
||||
for payload in (None, [], {'value_0': {'type': 'INT', 'value': 1}},
|
||||
{'value_1': {'type': 'LATENT', 'value': {}}},
|
||||
{'value_1': {'type': 'FLOAT', 'value': float('nan')}},
|
||||
{'value_1': {'type': 'STRING', 'value': None}},
|
||||
{'value_1': rt.encode_value(torch.tensor([float('inf')]), 'TENSOR')}):
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
rt.decode_result({**base, 'values': payload})
|
||||
|
||||
|
||||
def test_no_overrides_keeps_legacy_wire_and_does_not_encode_original_tensors():
|
||||
image = torch.zeros(1, 2, 2, 3)
|
||||
result = rt.normalize_worker_result(image)
|
||||
assert set(result) == {'image', 'mask', 'text'}
|
||||
with patch.object(rt, 'encode_value', wraps=rt.encode_value) as encode:
|
||||
wire = rt.encode_result(result)
|
||||
assert set(wire) == {'image', 'mask', 'text'}
|
||||
assert [call.args[1] for call in encode.call_args_list] == ['IMAGE', 'MASK', 'STRING']
|
||||
assert set(rt.decode_result(wire)) == {'image', 'mask', 'text'}
|
||||
|
||||
|
||||
def test_override_ancestors_survive_plan_and_worker_prompt():
|
||||
prompt = value_graph()
|
||||
frozen = copy.deepcopy(prompt)
|
||||
prepared, targets = prepare_image_list_execution(prompt)
|
||||
assert prepared == frozen and prompt == frozen
|
||||
assert targets == ['end']
|
||||
region = rt.validate_image_list_regions(prepared)[0]
|
||||
assert 'override' in region['branch_ids']
|
||||
worker = rt.build_worker_prompt(prepared, region, 'token', 'context', 0, 'http://master')
|
||||
assert worker['end']['inputs']['value_1'] == ['override', 0]
|
||||
assert worker['end']['inputs']['value_10'] == ['start', 5]
|
||||
assert 'source' not in worker
|
||||
for bad in ('value_0', 'value_11', 'values.value_1'):
|
||||
invalid = copy.deepcopy(prompt)
|
||||
invalid['end']['inputs'][bad] = ['start', 4]
|
||||
with pytest.raises(ValueError, match='slot'):
|
||||
rt.validate_image_list_regions(invalid)
|
||||
|
||||
|
||||
def test_missing_or_unexpected_result_override_fails_before_recording():
|
||||
prompt = value_graph()
|
||||
items = rt.normalize_image_list_inputs(torch.zeros(1, 2, 2, 3), values={'value_1': 'original'})
|
||||
context = rt.create_master_job('start', 'end', prompt, items, ['worker'], '')
|
||||
try:
|
||||
rt.register_assignment(context, 0, 'worker', 'token')
|
||||
for result_values in ({}, {'value_1': 'updated'},
|
||||
{'value_1': 'updated', 'value_10': 10, 'value_9': False}):
|
||||
with pytest.raises(ValueError, match='override'):
|
||||
rt.set_master_result(context, 0, 'token', {'image': items[0]['image'], 'values': result_values})
|
||||
assert rt.get_master_job(context).received_indexes == set()
|
||||
rt.set_master_result(context, 0, 'token', {'image': items[0]['image'],
|
||||
'values': {'value_1': 'updated', 'value_10': 10}})
|
||||
assert rt.get_master_job(context).received_indexes == {0}
|
||||
finally:
|
||||
rt.remove_master_job(context)
|
||||
|
||||
|
||||
def test_override_return_request_budget_is_aggregate():
|
||||
result = rt.normalize_worker_result(torch.zeros(1, 2, 2, 3), values={'value_1': ['x' * 500]})
|
||||
with patch.object(rt, 'MAX_REQUEST_BYTES', 400):
|
||||
with pytest.raises(ValueError, match='request'):
|
||||
rt.encode_request_payload({'result': rt.encode_result(result)})
|
||||
|
||||
|
||||
class TestResultHTTP:
|
||||
def test_missing_and_invalid_overrides_fail_through_real_http(self):
|
||||
import asyncio
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestClient, TestServer
|
||||
from test_image_list_integration_gaps import load_routes
|
||||
async def check():
|
||||
_module, routes = load_routes()
|
||||
app = web.Application()
|
||||
app.add_routes(routes)
|
||||
prompt = value_graph()
|
||||
items = rt.normalize_image_list_inputs(torch.zeros(1, 2, 2, 3), values={'value_1': 'original'})
|
||||
context = rt.create_master_job('start', 'end', prompt, items, ['worker'], '')
|
||||
token = rt.create_worker_task_token()
|
||||
rt.register_assignment(context, 0, 'worker', token)
|
||||
try:
|
||||
async with TestClient(TestServer(app)) as client:
|
||||
caps = await (await client.get('/distributed/image-list/capabilities')).json()
|
||||
assert caps['value_results'] is True and caps['protocol_version'] == 2
|
||||
result = rt.encode_result({'image': items[0]['image']})
|
||||
envelope = {'context_id': context, 'token': token, 'item_index': 0, 'result': result}
|
||||
response = await client.post('/distributed/image-list/result', json=envelope)
|
||||
assert response.status == 400
|
||||
assert 'missing' in (await response.json())['error']
|
||||
assert not rt.get_master_job(context).received_indexes
|
||||
result['values'] = {'value_1': {'type': 'INT', 'value': [1, 2]},
|
||||
'value_10': {'type': 'INT', 'value': 10}}
|
||||
response = await client.post('/distributed/image-list/result', json=envelope)
|
||||
assert response.status == 400
|
||||
assert not rt.get_master_job(context).received_indexes
|
||||
result['values']['value_1'] = {'type': 'STRING', 'value': 'new'}
|
||||
response = await client.post('/distributed/image-list/result', json=envelope)
|
||||
assert response.status == 200
|
||||
assert rt.get_master_job(context).results[0]['values'] == {'value_1': 'new', 'value_10': 10}
|
||||
finally:
|
||||
rt.remove_master_job(context)
|
||||
asyncio.run(check())
|
||||
@@ -94,7 +94,7 @@ async def check():
|
||||
assert await extension.get_node_list() == [MediaImportV1, MediaExportV1]
|
||||
imp = MediaImportV1.GET_NODE_INFO_V1()
|
||||
exp = MediaExportV1.GET_NODE_INFO_V1()
|
||||
assert imp['output'] == ['IMAGE','AUDIO','MASK','STRING','VIDEO'], imp
|
||||
assert imp['output'] == ['IMAGE','AUDIO','MASK','STRING','VIDEO','*'], imp
|
||||
assert exp['output_node'] and MediaExportV1.define_schema().is_input_list, exp
|
||||
with tempfile.TemporaryDirectory(prefix='typed-media-', dir=os.environ.get('TMPDIR')) as scratch:
|
||||
base = Path(scratch)
|
||||
@@ -152,7 +152,7 @@ async def check():
|
||||
mapped, missing, hidden = execution.get_input_data({'artifact_json':json.dumps(artifact),'filename':filename},MediaImportV1,'import')
|
||||
got = await execution._async_map_node_over_list('probe','import',MediaImportV1,mapped,MediaImportV1.FUNCTION,v3_data=hidden)
|
||||
got = await execution.resolve_map_node_over_list_results(got)
|
||||
assert len(got[0].result) == 5
|
||||
assert len(got[0].result) == 6
|
||||
assert all(x is None for n,x in enumerate(got[0].result) if n != list(SOCKET_KINDS).index(socket))
|
||||
bad = dict(artifact,sha256='0'*64)
|
||||
try: import_value(bad,filename)
|
||||
|
||||
@@ -1,314 +0,0 @@
|
||||
"""CPU-only typed collection and optional native V3 mapping acceptance."""
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOCKETS = ('images', 'audio', 'masks', 'texts', 'video')
|
||||
WORKERS = ['serverless:pool:1', 'serverless:pool:2']
|
||||
|
||||
|
||||
class Manager:
|
||||
def __init__(self, *, pending=False, owned=True):
|
||||
self.pending, self.owned = pending, owned
|
||||
self.cancelled = []
|
||||
|
||||
def owns(self, job, worker):
|
||||
return self.owned
|
||||
|
||||
def is_pending(self, job, worker):
|
||||
return self.pending
|
||||
|
||||
async def cancel(self, **kwargs):
|
||||
self.cancelled.append(kwargs)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def module(monkeypatch):
|
||||
package = '_media_collector_test'
|
||||
for name, path in ((package, ROOT), (package + '.nodes', ROOT / 'nodes'),
|
||||
(package + '.utils', ROOT / 'utils'),
|
||||
(package + '.serverless_bridge', ROOT / 'serverless_bridge')):
|
||||
stub = types.ModuleType(name)
|
||||
stub.__path__ = [str(path)]
|
||||
monkeypatch.setitem(sys.modules, name, stub)
|
||||
io = types.SimpleNamespace(ComfyNode=object, NodeOutput=lambda *items: types.SimpleNamespace(result=items))
|
||||
api = types.ModuleType('comfy_api.v0_0_2')
|
||||
api.io = io
|
||||
monkeypatch.setitem(sys.modules, 'comfy_api', types.ModuleType('comfy_api'))
|
||||
monkeypatch.setitem(sys.modules, 'comfy_api.v0_0_2', api)
|
||||
management = types.ModuleType('comfy.model_management')
|
||||
management.throw_exception_if_processing_interrupted = lambda: None
|
||||
comfy = types.ModuleType('comfy')
|
||||
comfy.model_management = management
|
||||
monkeypatch.setitem(sys.modules, 'comfy', comfy)
|
||||
monkeypatch.setitem(sys.modules, 'comfy.model_management', management)
|
||||
ps = types.SimpleNamespace(distributed_pending_jobs={}, distributed_jobs_lock=asyncio.Lock())
|
||||
server = types.ModuleType('server')
|
||||
server.PromptServer = types.SimpleNamespace(instance=ps)
|
||||
monkeypatch.setitem(sys.modules, 'server', server)
|
||||
helper = types.ModuleType(package + '.utils.async_helpers')
|
||||
helper.run_async_in_server_loop = asyncio.run
|
||||
monkeypatch.setitem(sys.modules, helper.__name__, helper)
|
||||
spec = importlib.util.spec_from_file_location(package + '.nodes.media_collector', ROOT / 'nodes/media_collector.py')
|
||||
loaded = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, loaded)
|
||||
spec.loader.exec_module(loaded)
|
||||
monkeypatch.setattr(loaded, 'POLL_SECONDS', 0.005)
|
||||
monkeypatch.setattr(loaded, 'START_GRACE_SECONDS', 0.025)
|
||||
loaded.test_server = ps
|
||||
loaded.test_management = management
|
||||
return loaded
|
||||
|
||||
|
||||
def native_media(offset=0):
|
||||
return {
|
||||
'images': [torch.full((1, 2 + offset, 3, 4), offset, dtype=torch.float64),
|
||||
torch.zeros((2, 1, 1, 3), dtype=torch.float16)],
|
||||
'audio': [{'waveform': torch.zeros((1, 1 + offset, 8), dtype=torch.float64),
|
||||
'sample_rate': 44100 + offset}],
|
||||
'masks': [torch.zeros((1, 2 + offset, 3), dtype=torch.float64)],
|
||||
'texts': ['', ' café\n🎥\r\n'],
|
||||
'video': [object()],
|
||||
}
|
||||
|
||||
|
||||
def setup(module, messages=(), *, manager=None):
|
||||
queue = asyncio.Queue()
|
||||
for message in messages:
|
||||
queue.put_nowait(message)
|
||||
ps = module.test_server
|
||||
ps.distributed_pending_jobs = {'job': queue, 'unrelated': asyncio.Queue()}
|
||||
ps.distributed_serverless = manager if manager is not None else Manager()
|
||||
return ps, queue
|
||||
|
||||
|
||||
def message(worker, media):
|
||||
return {'worker_id': worker, 'media': media, 'is_last': True}
|
||||
|
||||
|
||||
def collect(module, **kwargs):
|
||||
return module.DistributedMediaCollector.execute(**kwargs).result
|
||||
|
||||
|
||||
def remote(module, **kwargs):
|
||||
controls = dict(multi_job_id=['job'], enabled_worker_ids=[json.dumps(WORKERS)],
|
||||
expected_media=[json.dumps(['texts'])], delegate_only=[True])
|
||||
controls.update(kwargs)
|
||||
return collect(module, **controls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('controls', [{}, {'pass_through': [True]}, {'enabled_worker_ids': ['[]']}])
|
||||
def test_all_five_passthrough_preserves_lists_and_native_values(module, controls):
|
||||
values = native_media()
|
||||
result = collect(module, **values, **controls)
|
||||
assert len(result) == 5
|
||||
for index, socket in enumerate(SOCKETS):
|
||||
assert result[index] is values[socket]
|
||||
|
||||
|
||||
def test_reversed_arrivals_master_then_workers_then_items(module):
|
||||
master, first, second = native_media(), native_media(1), native_media(2)
|
||||
ps, _ = setup(module, [message(WORKERS[1], second), message(WORKERS[0], first)])
|
||||
result = remote(module, **master, delegate_only=[False], expected_media=[json.dumps(list(SOCKETS))])
|
||||
for index, socket in enumerate(SOCKETS):
|
||||
expected = master[socket] + first[socket] + second[socket]
|
||||
assert len(result[index]) == len(expected)
|
||||
assert all(actual is original for actual, original in zip(result[index], expected))
|
||||
assert list(ps.distributed_pending_jobs) == ['unrelated']
|
||||
assert ps.distributed_serverless.cancelled == []
|
||||
|
||||
|
||||
def test_delegate_only_and_empty_string(module):
|
||||
ps, _ = setup(module, [message(WORKERS[1], {'texts': ['b']}), message(WORKERS[0], {'texts': ['']})])
|
||||
result = remote(module)
|
||||
assert result == ([], [], [], ['', 'b'], [])
|
||||
assert 'job' not in ps.distributed_pending_jobs
|
||||
|
||||
|
||||
@pytest.mark.parametrize('kwargs', [{}, {'texts': []}, {'texts': ''}, {'texts': ['x'], 'is_worker': [True]},
|
||||
{'texts': ['x'], 'enabled_worker_ids': ['["direct-worker"]']},
|
||||
{'texts': ['x'], 'expected_media': ['["images"]']},
|
||||
{'texts': ['x'], 'pass_through': [True, False]}, {'delegate_only': True},
|
||||
{'texts': ['x'], 'enabled_worker_ids': ['["serverless:pool:1","serverless:pool:1"]']},
|
||||
{'texts': ['x'], 'load_balance': ['false']}])
|
||||
def test_rejects_missing_media_or_invalid_controls(module, kwargs):
|
||||
with pytest.raises((ValueError, RuntimeError)):
|
||||
collect(module, **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('bad', [
|
||||
{'worker_id': 'serverless:other:1', 'media': {'texts': ['x']}, 'is_last': True},
|
||||
{'worker_id': WORKERS[0], 'media': {'texts': ['x']}, 'is_last': False},
|
||||
{'worker_id': WORKERS[0], 'media': {'images': [object()]}, 'is_last': True},
|
||||
{'worker_id': WORKERS[0], 'media': {'texts': []}, 'is_last': True},
|
||||
{'worker_id': WORKERS[0], 'media': {'texts': 'x'}, 'is_last': True},
|
||||
{'worker_id': WORKERS[0], 'media': {'texts': [None]}, 'is_last': True},
|
||||
{'worker_id': WORKERS[0], 'media': {'texts': ['x'] * 65}, 'is_last': True},
|
||||
{'worker_id': WORKERS[0], 'media': {'texts': ['x'], 'unknown': ['x']}, 'is_last': True},
|
||||
{'worker_id': WORKERS[0], 'serverless_error': 'sanitized failure', 'is_last': True},
|
||||
])
|
||||
def test_rejects_bad_delivery_without_partial_output_and_cleans(module, bad):
|
||||
ps, _ = setup(module, [bad])
|
||||
with pytest.raises((ValueError, RuntimeError)):
|
||||
remote(module)
|
||||
assert list(ps.distributed_pending_jobs) == ['unrelated']
|
||||
assert ps.distributed_serverless.cancelled == [{'collector_job_id': 'job'}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('after_complete', [False, True])
|
||||
def test_duplicate_delivery_rejected_including_final_queue_drain(module, after_complete):
|
||||
first = message(WORKERS[0], {'texts': ['a']})
|
||||
second = message(WORKERS[1], {'texts': ['b']})
|
||||
messages = [first, second, first] if after_complete else [first, first, second]
|
||||
ps, _ = setup(module, messages)
|
||||
with pytest.raises(RuntimeError, match='[Dd]uplicate'):
|
||||
remote(module)
|
||||
assert 'job' not in ps.distributed_pending_jobs
|
||||
assert ps.distributed_serverless.cancelled == [{'collector_job_id': 'job'}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('manager', [None, Manager(owned=False), Manager(pending=False)])
|
||||
def test_missing_manager_or_finished_participant_fails_bounded(module, manager):
|
||||
ps, _ = setup(module)
|
||||
ps.distributed_serverless = manager
|
||||
with pytest.raises(RuntimeError, match='[Ss]erverless'):
|
||||
remote(module)
|
||||
assert 'job' not in ps.distributed_pending_jobs
|
||||
|
||||
|
||||
def test_control_error_also_cleans_exact_job(module):
|
||||
ps, _ = setup(module)
|
||||
with pytest.raises(ValueError):
|
||||
remote(module, expected_media='["unknown"]')
|
||||
assert list(ps.distributed_pending_jobs) == ['unrelated']
|
||||
assert ps.distributed_serverless.cancelled == [{'collector_job_id': 'job'}]
|
||||
|
||||
|
||||
def test_launch_race_and_pending_deadline_owned_by_manager(module, monkeypatch):
|
||||
ps, queue = setup(module)
|
||||
ps.distributed_serverless = None
|
||||
async def runner(coro):
|
||||
async def produce():
|
||||
await asyncio.sleep(0.01)
|
||||
ps.distributed_serverless = Manager(pending=True)
|
||||
await asyncio.sleep(0.035)
|
||||
queue.put_nowait(message(WORKERS[1], {'texts': ['b']}))
|
||||
queue.put_nowait(message(WORKERS[0], {'texts': ['a']}))
|
||||
task = asyncio.create_task(produce())
|
||||
result = await coro
|
||||
await task
|
||||
return result
|
||||
monkeypatch.setattr(module, 'run_async_in_server_loop', lambda coro: asyncio.run(runner(coro)))
|
||||
assert remote(module)[3] == ['a', 'b']
|
||||
assert ps.distributed_serverless.cancelled == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize('cancel_task', [False, True])
|
||||
def test_cancellation_while_blocked_cleans_and_propagates(module, monkeypatch, cancel_task):
|
||||
ps, _ = setup(module, manager=Manager(pending=True))
|
||||
class Interrupted(Exception):
|
||||
pass
|
||||
async def runner(coro):
|
||||
task = asyncio.create_task(coro)
|
||||
await asyncio.sleep(0.012)
|
||||
if cancel_task:
|
||||
task.cancel()
|
||||
else:
|
||||
def interrupt():
|
||||
raise Interrupted('stop')
|
||||
module.test_management.throw_exception_if_processing_interrupted = interrupt
|
||||
return await task
|
||||
monkeypatch.setattr(module, 'run_async_in_server_loop', lambda coro: asyncio.run(runner(coro)))
|
||||
with pytest.raises(asyncio.CancelledError if cancel_task else Interrupted):
|
||||
remote(module)
|
||||
assert list(ps.distributed_pending_jobs) == ['unrelated']
|
||||
assert ps.distributed_serverless.cancelled == [{'collector_job_id': 'job'}]
|
||||
|
||||
|
||||
def test_replaced_queue_not_removed(module, monkeypatch):
|
||||
ps, _ = setup(module, manager=Manager(pending=True))
|
||||
replacement = asyncio.Queue()
|
||||
async def runner(coro):
|
||||
task = asyncio.create_task(coro)
|
||||
await asyncio.sleep(0.012)
|
||||
ps.distributed_pending_jobs['job'] = replacement
|
||||
return await task
|
||||
monkeypatch.setattr(module, 'run_async_in_server_loop', lambda coro: asyncio.run(runner(coro)))
|
||||
with pytest.raises(RuntimeError, match='queue'):
|
||||
remote(module)
|
||||
assert ps.distributed_pending_jobs['job'] is replacement
|
||||
|
||||
|
||||
def test_native_v3_schema_and_list_mapping_cpu():
|
||||
core = os.environ.get('COMFYUI_SOURCE_ROOT')
|
||||
if not core:
|
||||
pytest.skip('Set COMFYUI_SOURCE_ROOT for CPU-only native V3 mapping')
|
||||
probe = r'''
|
||||
import asyncio, importlib.util, sys, types
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
import comfy.cli_args
|
||||
comfy.cli_args.args.cpu = True
|
||||
comfy.cli_args.args.disable_assets = True
|
||||
import execution
|
||||
import torch
|
||||
import server
|
||||
# No HTTP listener or PromptServer bootstrap is needed for schema/list mapping.
|
||||
server.PromptServer.instance = types.SimpleNamespace()
|
||||
root = Path(sys.argv[2])
|
||||
for name, path in [('typed_collector', root), ('typed_collector.nodes', root/'nodes'),
|
||||
('typed_collector.utils', root/'utils'),
|
||||
('typed_collector.serverless_bridge', root/'serverless_bridge')]:
|
||||
package = types.ModuleType(name)
|
||||
package.__path__ = [str(path)]
|
||||
sys.modules[name] = package
|
||||
spec = importlib.util.spec_from_file_location('typed_collector.nodes.media_collector', root/'nodes/media_collector.py')
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
cls = module.DistributedMediaCollector
|
||||
info = cls.GET_NODE_INFO_V1()
|
||||
assert info['output'] == ['IMAGE','AUDIO','MASK','STRING','VIDEO'], info
|
||||
assert info['output_is_list'] == [True]*5, info
|
||||
assert cls.define_schema().is_input_list and cls.define_schema().accept_all_inputs
|
||||
for name, kind in zip(['images','audio','masks','texts','video'], info['output']):
|
||||
field = info['input']['optional'][name]
|
||||
assert field[0] == kind and field[1]['forceInput'] is True, field
|
||||
assert info['input']['required']['load_balance'][1]['default'] is False
|
||||
async def check():
|
||||
values = {'load_balance':False, 'pass_through':True}
|
||||
prepared, missing, hidden = execution.get_input_data(values, cls, unique_id='probe')
|
||||
assert not missing, missing
|
||||
image = torch.zeros((1,2,3,4), dtype=torch.float64)
|
||||
audio = {'waveform':torch.zeros((1,2,7)), 'sample_rate':48000}
|
||||
mask = torch.ones((1,2,3), dtype=torch.float64)
|
||||
from comfy_api.input_impl import VideoFromComponents
|
||||
from comfy_api.util import VideoComponents
|
||||
from fractions import Fraction
|
||||
video = VideoFromComponents(VideoComponents(images=image[:,:,:,:3].float(), frame_rate=Fraction(24,1)))
|
||||
native = {'images':[image], 'audio':[audio], 'masks':[mask], 'texts':[''], 'video':[video]}
|
||||
prepared.update(native)
|
||||
results = await execution._async_map_node_over_list('cpu','probe',cls,prepared,cls.FUNCTION,v3_data=hidden)
|
||||
results = await execution.resolve_map_node_over_list_results(results)
|
||||
assert len(results) == 1
|
||||
for index, name in enumerate(native):
|
||||
assert results[0].result[index] is native[name]
|
||||
assert results[0].result[4][0] is video
|
||||
print('MEDIA_COLLECTOR_CPU_OK')
|
||||
asyncio.run(check())
|
||||
'''
|
||||
env = {**os.environ, 'CUDA_VISIBLE_DEVICES': '', 'HIP_VISIBLE_DEVICES': '', 'PYTHONDONTWRITEBYTECODE': '1'}
|
||||
env.pop('PYTHONPATH', None)
|
||||
result = subprocess.run([sys.executable, '-c', probe, core, str(ROOT)], env=env,
|
||||
capture_output=True, text=True, timeout=90)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert 'MEDIA_COLLECTOR_CPU_OK' in result.stdout
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Opt-in, isolated CPU acceptance of the complete typed-media data path.
|
||||
"""Opt-in CPU acceptance of Collector IMAGE/AUDIO and the five-kind bridge.
|
||||
|
||||
COMFYUI_SOURCE_ROOT must name an existing compatible checkout; run with that
|
||||
checkout's Python environment. No installation, normal instance, or provider is
|
||||
@@ -42,6 +42,7 @@ def test_typed_media_end_to_end_cpu():
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
for marker in ("NATIVE_INPUTS_AND_DIRECT_EXPORT_REFS_OK", "REAL_HTTP_BRIDGE_RESULTS_OK",
|
||||
"MASTER_AND_DELEGATE_ORDER_OK", "ATOMIC_DECODE_REJECTION_OK",
|
||||
"IMAGE_TRANSFER_HTTP_OK",
|
||||
"MEDIA_END_TO_END_CPU_OK"):
|
||||
assert marker in result.stdout, result.stdout + result.stderr
|
||||
print(result.stdout)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Pure integration of participant identity, typed compilation and delegate pruning."""
|
||||
"""Participant identity, typed compilation and legacy Collector pruning."""
|
||||
import copy
|
||||
import importlib
|
||||
import json
|
||||
@@ -16,17 +16,17 @@ for suffix in ("", ".api", ".api.orchestration", ".utils", ".serverless_bridge")
|
||||
sys.modules[module.__name__] = module
|
||||
pt = importlib.import_module(PACKAGE + ".api.orchestration.prompt_transform")
|
||||
compiler = importlib.import_module(PACKAGE + ".api.orchestration.serverless_media_prompt")
|
||||
contract = importlib.import_module(PACKAGE + ".serverless_bridge.contract")
|
||||
SOCKETS = ("images", "audio")
|
||||
|
||||
|
||||
def prompt():
|
||||
def prompt(sockets=SOCKETS):
|
||||
nodes = {
|
||||
"source": {"class_type": "MediaSource", "inputs": {}},
|
||||
"collector": {"class_type": "DistributedMediaCollector", "inputs": {
|
||||
socket: ["source", index] for index, socket in enumerate(contract.SOCKET_KINDS)}},
|
||||
"collector": {"class_type": "DistributedCollector", "inputs": {
|
||||
socket: ["source", SOCKETS.index(socket)] for socket in sockets}},
|
||||
}
|
||||
for index, socket in enumerate(contract.SOCKET_KINDS):
|
||||
nodes["sink:" + socket] = {"class_type": "Capture", "inputs": {"value": ["collector", index]}}
|
||||
for socket in sockets:
|
||||
nodes["sink:" + socket] = {"class_type": "Capture", "inputs": {"value": ["collector", SOCKETS.index(socket)]}}
|
||||
return nodes
|
||||
|
||||
|
||||
@@ -38,47 +38,41 @@ def prepare(graph, participant, delegate):
|
||||
["serverless:pool:1", "serverless:pool:2"], jobs, "http://master", delegate, index)
|
||||
|
||||
|
||||
def test_typed_worker_preparation_compiles_all_bindings_without_source_mutation():
|
||||
def test_typed_worker_preparation_compiles_both_bindings_without_source_mutation():
|
||||
graph = prompt()
|
||||
before = copy.deepcopy(graph)
|
||||
worker = prepare(graph, "serverless:pool:2", True)
|
||||
compiled, bindings = compiler.compile_media_prompt(worker, "request:2")
|
||||
assert len(bindings) == 5
|
||||
assert len(bindings) == 2
|
||||
assert {row["participant_id"] for row in bindings} == {"serverless:pool:2"}
|
||||
assert {row["job_id"] for row in bindings} == {"run_collector"}
|
||||
assert {row["socket"] for row in bindings} == set(contract.SOCKET_KINDS)
|
||||
assert {row["socket"] for row in bindings} == set(SOCKETS)
|
||||
assert set(compiled) == {"source"} | {row["node_id"] for row in bindings}
|
||||
assert graph == before
|
||||
|
||||
|
||||
def test_typed_delegate_master_keeps_outputs_without_sources_or_placeholders():
|
||||
graph = prompt()
|
||||
@pytest.mark.parametrize("sockets", [("images",), ("audio",), SOCKETS])
|
||||
def test_delegate_master_keeps_collector_socket_contract_and_legacy_placeholder(sockets):
|
||||
graph = prompt(sockets)
|
||||
prepared = prepare(graph, "master", True)
|
||||
before = copy.deepcopy(prepared)
|
||||
delegated = pt.prepare_delegate_master_prompt(prepared, ["collector"])
|
||||
assert set(delegated) == {"collector"} | {"sink:" + s for s in contract.SOCKET_KINDS}
|
||||
assert "source" not in delegated
|
||||
assert {"collector"} | {"sink:" + s for s in sockets} <= set(delegated)
|
||||
inputs = delegated["collector"]["inputs"]
|
||||
assert not set(contract.SOCKET_KINDS) & set(inputs)
|
||||
assert json.loads(inputs["expected_media"]) == list(contract.SOCKET_KINDS)
|
||||
assert "audio" not in inputs and "expected_media" not in inputs
|
||||
placeholders = [key for key, node in delegated.items() if node['class_type'] == 'DistributedEmptyImage']
|
||||
if "images" in sockets:
|
||||
assert len(placeholders) == 1
|
||||
assert inputs['images'] == [placeholders[0], 0]
|
||||
else:
|
||||
assert not placeholders and 'images' not in inputs
|
||||
assert inputs["delegate_only"] is True
|
||||
assert json.loads(inputs["enabled_worker_ids"]) == ["serverless:pool:1", "serverless:pool:2"]
|
||||
assert prepared == before
|
||||
|
||||
|
||||
def test_text_source_is_not_retained_as_lightweight_delegate_scalar(monkeypatch):
|
||||
graph = {
|
||||
"text": {"class_type": "PrimitiveString", "inputs": {"value": "local text"}},
|
||||
"collector": {"class_type": "DistributedMediaCollector", "inputs": {"texts": ["text", 0]}},
|
||||
"sink": {"class_type": "Capture", "inputs": {"value": ["collector", 3]}},
|
||||
}
|
||||
monkeypatch.setattr(pt, "_delegate_master_input_is_safe_scalar", lambda *args: True)
|
||||
monkeypatch.setattr(pt, "_is_delegate_master_retained_upstream_node", lambda *args: True)
|
||||
delegated = pt.prepare_delegate_master_prompt(prepare(graph, "master", True), ["collector"])
|
||||
assert "text" not in delegated
|
||||
assert "texts" not in delegated["collector"]["inputs"]
|
||||
assert json.loads(delegated["collector"]["inputs"]["expected_media"]) == ["texts"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("socket", contract.SOCKET_KINDS)
|
||||
@pytest.mark.parametrize("socket", SOCKETS)
|
||||
def test_participating_master_retains_exact_media_links(socket):
|
||||
graph = prompt()
|
||||
original = copy.deepcopy(graph)
|
||||
|
||||
@@ -35,11 +35,11 @@ def node(class_type, **inputs):
|
||||
return {"class_type": class_type, "inputs": inputs}
|
||||
|
||||
|
||||
def text_prompt():
|
||||
def collector_prompt():
|
||||
return {
|
||||
"text": node("PrimitiveString", value=" café雪\n"),
|
||||
"collector": node("DistributedMediaCollector", texts=["text", 0], load_balance=False),
|
||||
"save": node("TextSink", texts=["collector", 3]),
|
||||
"source": node("EmptyImage", width=32, height=32, batch_size=1, color=0),
|
||||
"collector": node("DistributedCollector", images=["source", 0], load_balance=False),
|
||||
"save": node("SaveImage", images=["collector", 0], filename_prefix="local"),
|
||||
}
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ def media_paths(tmp_path, monkeypatch):
|
||||
|
||||
def run(h, prompt=None, **kwargs):
|
||||
return asyncio.run(h.q.orchestrate_distributed_execution(
|
||||
prompt if prompt is not None else text_prompt(), None, "client", **kwargs))
|
||||
prompt if prompt is not None else collector_prompt(), None, "client", **kwargs))
|
||||
|
||||
|
||||
def assert_no_dispatch(h, *, no_probe=False):
|
||||
@@ -117,10 +117,11 @@ def test_real_compile_pack_serialize_and_canonical_bindings(harness, media_paths
|
||||
image_path = media_paths / "picture.png"
|
||||
Image.new("RGBA", (2, 3), color=(17, 29, 41, 53)).save(image_path)
|
||||
image_bytes = image_path.read_bytes()
|
||||
prompt = text_prompt()
|
||||
prompt = collector_prompt()
|
||||
prompt["image"] = node("LoadImage", image="picture.png")
|
||||
prompt["collector"]["inputs"].update(images=["image", 0], masks=["image", 1])
|
||||
prompt["collector2"] = node("DistributedMediaCollector", texts=["text", 0])
|
||||
prompt["audio"] = node("GeneratedAudio")
|
||||
prompt["collector"]["inputs"].update(images=["image", 0], audio=["audio", 0])
|
||||
prompt["collector2"] = node("DistributedCollector", audio=["audio", 0])
|
||||
prompt["unused"] = node("PreviewImage", images=["image", 0])
|
||||
source = copy.deepcopy(prompt)
|
||||
with patch.object(h.compiler, "compile_media_prompt", wraps=h.compiler.compile_media_prompt) as compile_call, \
|
||||
@@ -145,8 +146,8 @@ def test_real_compile_pack_serialize_and_canonical_bindings(harness, media_paths
|
||||
assert media["outputs"] == job["manifest"]
|
||||
assert media["outputs"] is not job["manifest"]
|
||||
assert media["artifacts"] == media["inputs"] == []
|
||||
assert {b["socket"] for b in job["manifest"]} == {"texts", "images", "masks"}
|
||||
assert len(job["manifest"]) == 4
|
||||
assert {b["socket"] for b in job["manifest"]} == {"images", "audio"}
|
||||
assert len(job["manifest"]) == 3
|
||||
assert base64.b64decode(job["images"][0]["image"]) == image_bytes
|
||||
assert job["workflow"]["image"]["inputs"]["image"] == job["images"][0]["name"]
|
||||
assert not ({"save", "unused", "collector", "collector2"} & set(job["workflow"]))
|
||||
@@ -164,40 +165,50 @@ def test_real_compile_pack_serialize_and_canonical_bindings(harness, media_paths
|
||||
assert json.loads(controls["binding_json"]) == binding
|
||||
assert controls["request_id"] == media["request_id"]
|
||||
assert master["collector"]["inputs"]["delegate_only"] is delegate
|
||||
assert json.loads(master["collector"]["inputs"]["expected_media"]) == ["images", "masks", "texts"]
|
||||
assert "expected_media" not in master["collector"]["inputs"]
|
||||
if delegate:
|
||||
assert "text" not in master and "image" not in master
|
||||
assert not any(name in master["collector"]["inputs"] for name in ("images", "audio", "masks", "texts", "video"))
|
||||
assert not any(n["class_type"] == "DistributedEmptyImage" for n in master.values())
|
||||
assert not {"source", "audio"} & set(master)
|
||||
# LoadImage is an existing explicitly retained lightweight source; the
|
||||
# Collector is nevertheless detached from it and uses the placeholder.
|
||||
assert master["image"] == prompt["image"]
|
||||
assert "audio" not in master["collector"]["inputs"]
|
||||
placeholder = master["collector"]["inputs"]["images"][0]
|
||||
assert master[placeholder]["class_type"] == "DistributedEmptyImage"
|
||||
else:
|
||||
assert master["collector"]["inputs"]["texts"] == ["text", 0]
|
||||
assert "text" in master and "image" in master
|
||||
assert master["collector"]["inputs"]["audio"] == ["audio", 0]
|
||||
assert "audio" in master and "image" in master
|
||||
|
||||
|
||||
def test_delegate_disconnects_every_media_socket(harness):
|
||||
prompt = text_prompt()
|
||||
for index, name in enumerate(("images", "audio", "masks", "video")):
|
||||
source_id = "source-" + name
|
||||
prompt[source_id] = node("GeneratedMedia", seed=index)
|
||||
prompt["collector"]["inputs"][name] = [source_id, 0]
|
||||
@pytest.mark.parametrize("audio_only", [False, True])
|
||||
def test_delegate_prunes_image_audio_sources_and_audio_only_needs_no_placeholder(harness, audio_only):
|
||||
prompt = collector_prompt()
|
||||
prompt["audio"] = node("GeneratedAudio")
|
||||
prompt["collector"]["inputs"]["audio"] = ["audio", 0]
|
||||
if audio_only:
|
||||
del prompt["collector"]["inputs"]["images"]
|
||||
prompt["save"] = node("AudioSink", audio=["collector", 1])
|
||||
original = copy.deepcopy(prompt)
|
||||
run(harness, prompt)
|
||||
master = harness.q.queue_prompt_payload.call_args.args[0]
|
||||
inputs = master["collector"]["inputs"]
|
||||
sockets = {"images", "audio", "masks", "texts", "video"}
|
||||
assert set(json.loads(inputs["expected_media"])) == sockets
|
||||
assert not sockets.intersection(inputs)
|
||||
assert set(master) == {"collector", "save"}
|
||||
assert "audio" not in inputs and "expected_media" not in inputs
|
||||
assert "source" not in master and "audio" not in master
|
||||
if audio_only:
|
||||
assert "images" not in inputs and set(master) == {"collector", "save"}
|
||||
else:
|
||||
assert master[inputs["images"][0]]["class_type"] == "DistributedEmptyImage"
|
||||
for job in harness.manager.start.call_args.args[2]:
|
||||
assert {b["socket"] for b in job["manifest"]} == sockets
|
||||
assert {"text", "source-images", "source-audio", "source-masks", "source-video"} <= set(job["workflow"])
|
||||
assert {b["socket"] for b in job["manifest"]} == ({"audio"} if audio_only else {"images", "audio"})
|
||||
assert "audio" in job["workflow"]
|
||||
assert ("source" in job["workflow"]) is not audio_only
|
||||
assert harness.validator.validate_prepared_job(job) is True
|
||||
assert prompt == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selected", ["master", "worker"])
|
||||
def test_typed_load_balance_selects_master_or_one_serverless_slot(harness, selected):
|
||||
h = harness
|
||||
prompt = text_prompt()
|
||||
prompt = collector_prompt()
|
||||
prompt["collector"]["inputs"]["load_balance"] = "true"
|
||||
h.q.select_least_busy_worker.side_effect = lambda candidates, **kw: next(
|
||||
w for w in candidates if (w["id"] == "master") == (selected == "master"))
|
||||
@@ -208,23 +219,24 @@ def test_typed_load_balance_selects_master_or_one_serverless_slot(harness, selec
|
||||
assert result[2] == 0
|
||||
h.manager.start.assert_not_awaited()
|
||||
assert master["collector"]["inputs"]["delegate_only"] is False
|
||||
assert "text" in master
|
||||
assert "source" in master
|
||||
else:
|
||||
assert result[2] == 1
|
||||
assert len(h.manager.start.call_args.args[2]) == 1
|
||||
assert master["collector"]["inputs"]["delegate_only"] is True
|
||||
assert "text" not in master
|
||||
assert "source" not in master
|
||||
|
||||
|
||||
@pytest.mark.parametrize("profile", [None, "legacy-image-v1", PROFILE])
|
||||
def test_explicit_profile_compatibility_rejected_before_probe(harness, profile):
|
||||
@pytest.mark.parametrize("profile", [None, "legacy-image-v1"])
|
||||
def test_exact_rejects_legacy_pool_before_probe(harness, profile):
|
||||
h = harness
|
||||
prompt = legacy_prompt() if profile == PROFILE else text_prompt()
|
||||
prompt = collector_prompt()
|
||||
prompt["collector"]["inputs"]["transfer_format"] = "exact"
|
||||
if profile is None:
|
||||
h.pool.pop("media_profile")
|
||||
else:
|
||||
h.pool["media_profile"] = profile
|
||||
with pytest.raises(ValueError, match="requires|require"):
|
||||
with pytest.raises(ValueError, match="exact requires comfy-media-v1"):
|
||||
run(h, prompt)
|
||||
assert_no_dispatch(h, no_probe=True)
|
||||
|
||||
@@ -239,19 +251,26 @@ def test_invalid_explicit_profiles_fail_closed(harness, profile):
|
||||
|
||||
@pytest.mark.parametrize("direct_type", ["local", "remote", "cloud"])
|
||||
@pytest.mark.parametrize("explicit", [False, True])
|
||||
def test_direct_workers_never_silently_skipped_or_fallback(harness, direct_type, explicit):
|
||||
def test_direct_workers_participate_alongside_typed_pools(harness, direct_type, explicit):
|
||||
h = harness
|
||||
h.config["workers"].append({"id": "direct", "type": direct_type, "host": "offline.invalid",
|
||||
"enabled": not explicit, "port": 8188})
|
||||
prompt = text_prompt()
|
||||
prompt["collector"]["inputs"]["load_balance"] = True
|
||||
kwargs = {"enabled_worker_ids": ["pool", "direct"]} if explicit else {}
|
||||
with pytest.raises(ValueError, match="direct workers"):
|
||||
run(h, prompt, delegate_master=False, **kwargs)
|
||||
assert_no_dispatch(h, no_probe=True)
|
||||
prompt = collector_prompt()
|
||||
original = copy.deepcopy(prompt)
|
||||
assert run(h, prompt, delegate_master=False, **kwargs)[2] == 3
|
||||
assert prompt == original
|
||||
h.q.dispatch_worker_prompt.assert_awaited_once()
|
||||
master = h.q.queue_prompt_payload.call_args.args[0]
|
||||
assert json.loads(master["collector"]["inputs"]["enabled_worker_ids"]) == [
|
||||
"serverless:pool:1", "serverless:pool:2", "direct"]
|
||||
jobs = h.manager.start.call_args.args[2]
|
||||
assert len(jobs) == 2 and all(j["media"]["profile"] == PROFILE for j in jobs)
|
||||
if direct_type != "local":
|
||||
h.q.sync_worker_media.assert_awaited_once()
|
||||
|
||||
|
||||
def test_disabled_or_unselected_incompatible_workers_do_not_block(harness):
|
||||
def test_disabled_or_unselected_workers_do_not_participate(harness):
|
||||
h = harness
|
||||
h.config["workers"] += [
|
||||
{"id": "direct", "type": "local", "enabled": False},
|
||||
@@ -262,27 +281,32 @@ def test_disabled_or_unselected_incompatible_workers_do_not_block(harness):
|
||||
assert run(h, enabled_worker_ids=["pool"])[2] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("other", ["DistributedCollector", "UltimateSDUpscaleDistributed", "DistributedImageListStart"])
|
||||
@pytest.mark.parametrize("workers", [True, False])
|
||||
def test_mixed_collectors_or_image_list_rejected_even_master_only(harness, other, workers):
|
||||
h = harness
|
||||
def test_image_list_cannot_mix_with_collector_even_master_only(harness, workers):
|
||||
if not workers:
|
||||
h.config["workers"] = []
|
||||
prompt = text_prompt()
|
||||
prompt["incompatible"] = node(other)
|
||||
harness.config["workers"] = []
|
||||
prompt = collector_prompt()
|
||||
prompt["incompatible"] = node("DistributedImageListStart")
|
||||
with pytest.raises(ValueError, match="cannot be mixed"):
|
||||
run(h, prompt, delegate_master=False)
|
||||
assert_no_dispatch(h, no_probe=True)
|
||||
run(harness, prompt, delegate_master=False)
|
||||
assert_no_dispatch(harness, no_probe=True)
|
||||
|
||||
|
||||
def test_mixed_pool_profiles_rejected_before_load_balancing(harness):
|
||||
def test_mixed_direct_typed_and_legacy_profiles_keep_each_wire_contract(harness):
|
||||
h = harness
|
||||
h.config["workers"].append(dict(h.pool, id="legacy", media_profile="legacy-image-v1"))
|
||||
prompt = text_prompt()
|
||||
prompt["collector"]["inputs"]["load_balance"] = True
|
||||
with pytest.raises(ValueError, match="requires"):
|
||||
run(h, prompt, delegate_master=False)
|
||||
assert_no_dispatch(h, no_probe=True)
|
||||
h.config["workers"] += [dict(h.pool, id="legacy", media_profile="legacy-image-v1"),
|
||||
{"id": "direct", "type": "local", "enabled": True}]
|
||||
assert run(h, collector_prompt())[2] == 5
|
||||
h.q.dispatch_worker_prompt.assert_awaited_once()
|
||||
jobs = h.manager.start.call_args.args[2]
|
||||
assert len(jobs) == 4
|
||||
for job in jobs:
|
||||
if job["worker"]["media_profile"] == PROFILE:
|
||||
assert job["media"]["outputs"] == job["manifest"]
|
||||
assert h.validator.validate_prepared_job(job) is True
|
||||
else:
|
||||
assert "media" not in job
|
||||
assert any(n["class_type"] == "SaveImage" for n in job["workflow"].values())
|
||||
|
||||
|
||||
def test_typed_master_only_is_supported_without_serverless_work(harness):
|
||||
@@ -291,27 +315,23 @@ def test_typed_master_only_is_supported_without_serverless_work(harness):
|
||||
assert run(h, delegate_master=False)[2] == 0
|
||||
master = h.q.queue_prompt_payload.call_args.args[0]
|
||||
assert master["collector"]["inputs"]["enabled_worker_ids"] == "[]"
|
||||
assert master["collector"]["inputs"]["texts"] == ["text", 0]
|
||||
assert master["collector"]["inputs"]["images"] == ["source", 0]
|
||||
h.manager.start.assert_not_awaited()
|
||||
h.q.dispatch_worker_prompt.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offline", [False, True])
|
||||
def test_typed_delegate_requires_available_worker_without_master_fallback(harness, offline):
|
||||
def test_collector_delegate_without_workers_preserves_master_fallback(harness, offline):
|
||||
h = harness
|
||||
if offline:
|
||||
h.q.select_active_workers.side_effect = lambda *a, **kw: ([], False)
|
||||
else:
|
||||
h.config["workers"] = []
|
||||
with pytest.raises(RuntimeError, match="comfy-media-v1"):
|
||||
run(h)
|
||||
assert_no_dispatch(h)
|
||||
|
||||
|
||||
def test_unknown_explicit_typed_worker_is_rejected(harness):
|
||||
with pytest.raises(ValueError, match="Unknown explicitly selected"):
|
||||
run(harness, enabled_worker_ids=["unknown"], delegate_master=False)
|
||||
assert_no_dispatch(harness, no_probe=True)
|
||||
assert run(h)[2] == 0
|
||||
master = h.q.queue_prompt_payload.call_args.args[0]
|
||||
assert master["collector"]["inputs"]["delegate_only"] is False
|
||||
assert master["collector"]["inputs"]["images"] == ["source", 0]
|
||||
h.manager.start.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", [" pool ", "pool white", "pool\n"])
|
||||
@@ -323,9 +343,9 @@ def test_pool_media_identity_is_not_repaired(harness, bad):
|
||||
|
||||
|
||||
def test_integer_source_node_identity_is_not_normalized(harness):
|
||||
prompt = text_prompt()
|
||||
prompt[1] = prompt.pop("text")
|
||||
prompt["collector"]["inputs"]["texts"] = [1, 0]
|
||||
prompt = collector_prompt()
|
||||
prompt[1] = prompt.pop("source")
|
||||
prompt["collector"]["inputs"]["images"] = [1, 0]
|
||||
with pytest.raises(ValueError, match="identifier"):
|
||||
run(harness, prompt)
|
||||
assert_no_dispatch(harness, no_probe=True)
|
||||
@@ -334,18 +354,18 @@ def test_integer_source_node_identity_is_not_normalized(harness):
|
||||
@pytest.mark.parametrize("broken", ["dangling-unused", "literal", "missing-file", "bad-loader-controls", "oversize"])
|
||||
def test_compilation_packing_and_size_errors_precede_all_dispatch(harness, media_paths, broken):
|
||||
h = harness
|
||||
prompt = text_prompt()
|
||||
prompt = collector_prompt()
|
||||
if broken == "dangling-unused":
|
||||
prompt["unused"] = node("PreviewImage", images=["missing", 0])
|
||||
elif broken == "literal":
|
||||
prompt["collector"]["inputs"]["texts"] = "must use a linked string node"
|
||||
prompt["collector"]["inputs"]["images"] = "must use a linked image node"
|
||||
elif broken in ("missing-file", "bad-loader-controls"):
|
||||
prompt["audio"] = node("LoadAudio", audio="missing.wav")
|
||||
prompt["collector"]["inputs"]["audio"] = ["audio", 0]
|
||||
if broken == "bad-loader-controls":
|
||||
prompt["audio"]["inputs"]["duration"] = 5
|
||||
else:
|
||||
prompt["text"]["inputs"]["value"] = "雪" * 1000
|
||||
prompt["source"]["inputs"]["label"] = "雪" * 1000
|
||||
original = copy.deepcopy(prompt)
|
||||
with patch.object(h.client, "MAX_REQUEST_BYTES", 100 if broken == "oversize" else h.client.MAX_REQUEST_BYTES):
|
||||
with pytest.raises((ValueError, OSError)):
|
||||
@@ -418,9 +438,9 @@ def test_master_queue_failure_cleans_queues_and_never_starts_manager(harness):
|
||||
|
||||
def test_manager_setup_failure_emits_one_terminal_per_collector_participant(harness):
|
||||
h = harness
|
||||
prompt = text_prompt()
|
||||
prompt["image"] = node("EmptyImage", width=2, height=2, batch_size=1, color=0)
|
||||
prompt["collector"]["inputs"]["images"] = ["image", 0]
|
||||
prompt = collector_prompt()
|
||||
prompt["audio"] = node("GeneratedAudio")
|
||||
prompt["collector"]["inputs"]["audio"] = ["audio", 0]
|
||||
h.manager.start.side_effect = ValueError("journal failure")
|
||||
with pytest.raises(ValueError, match="journal failure"):
|
||||
run(h, prompt)
|
||||
@@ -481,6 +501,7 @@ import comfy.cli_args
|
||||
comfy.cli_args.args.cpu = True
|
||||
import torch
|
||||
|
||||
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('CPU audio integration may not initialize a GPU')
|
||||
torch.cuda.is_available = lambda: False
|
||||
@@ -507,7 +528,7 @@ raise SystemExit(pytest.main([sys.argv[2], '-q', '-p', 'no:cacheprovider', '-k',
|
||||
stream.setframerate(8000)
|
||||
stream.writeframes(pcm.numpy().tobytes())
|
||||
original_bytes = path.read_bytes()
|
||||
prompt = text_prompt()
|
||||
prompt = collector_prompt()
|
||||
prompt["audio"] = node("LoadAudio", audio="tone.wav")
|
||||
prompt["collector"]["inputs"]["audio"] = ["audio", 0]
|
||||
original = copy.deepcopy(prompt)
|
||||
|
||||
@@ -6,16 +6,16 @@ import pytest
|
||||
|
||||
from test_serverless_media_prompt import compiler, _prompt
|
||||
from test_media_handler import handler
|
||||
from test_media_collector import module, collect, native_media
|
||||
from test_collector_typed_media import collector_module, native_media, assert_exact
|
||||
|
||||
FORMATS = ('exact', 'legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1')
|
||||
FORMATS = ('legacy_png', 'png', 'ffv1', 'jpeg', 'webp', 'h264', 'h265', 'av1', 'exact')
|
||||
|
||||
|
||||
def compile_request(fmt='exact', quality=90):
|
||||
def compile_request(fmt='legacy_png', quality=90):
|
||||
prompt = _prompt()
|
||||
prompt['source:1'] = {'class_type': 'Source', 'inputs': {}}
|
||||
prompt['collector:1']['inputs'].update(transfer_format=fmt, transfer_quality=quality,
|
||||
masks=['source:1', 1], texts=['source:1', 2])
|
||||
audio=['source:1', 1])
|
||||
original = copy.deepcopy(prompt)
|
||||
workflow, outputs = compiler.compile_media_prompt(prompt, 'request:1')
|
||||
assert prompt == original
|
||||
@@ -27,16 +27,23 @@ def compile_request(fmt='exact', quality=90):
|
||||
@pytest.mark.parametrize('fmt', FORMATS)
|
||||
def test_compiler_and_handler_preserve_image_policy_only(fmt):
|
||||
data = compile_request(fmt, 73)
|
||||
assert {b['socket'] for b in data['media']['outputs']} == {'images', 'audio'}
|
||||
for binding in data['media']['outputs']:
|
||||
inputs = data['workflow'][binding['node_id']]['inputs']
|
||||
if binding['kind'] == 'IMAGE' and fmt != 'exact':
|
||||
assert inputs['transfer_format'] == fmt
|
||||
assert inputs['transfer_format'] == ('png' if fmt == 'legacy_png' else fmt)
|
||||
assert inputs['transfer_quality'] == 73
|
||||
else:
|
||||
assert 'transfer_format' not in inputs and 'transfer_quality' not in inputs
|
||||
handler._validate_request(data)
|
||||
|
||||
|
||||
def test_saved_collector_defaults_compile_to_png_not_exact():
|
||||
workflow, bindings = compiler.compile_media_prompt(_prompt(), 'request:1')
|
||||
inputs = workflow[bindings[0]['node_id']]['inputs']
|
||||
assert (inputs['transfer_format'], inputs['transfer_quality']) == ('png', 90)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fmt,quality', [('JPEG',90),('',90),('jpeg',True),('jpeg',0),
|
||||
('jpeg',101),('jpeg','90'),(['source:1',0],90)])
|
||||
def test_compiler_rejects_invalid_or_linked_controls(fmt, quality):
|
||||
@@ -54,23 +61,36 @@ def test_handler_rejects_tampered_policy(field, value):
|
||||
handler._validate_request(data)
|
||||
|
||||
|
||||
def test_handler_rejects_image_controls_on_mask_export():
|
||||
data = compile_request()
|
||||
binding = next(b for b in data['media']['outputs'] if b['kind'] == 'MASK')
|
||||
data['workflow'][binding['node_id']]['inputs']['transfer_format'] = 'jpeg'
|
||||
@pytest.mark.parametrize('socket,kind', [('audio', 'AUDIO'), ('masks', 'MASK'), ('texts', 'STRING'), ('video', 'VIDEO')])
|
||||
def test_handler_rejects_image_controls_on_nonimage_bridge_exports(socket, kind):
|
||||
# The bridge still supports all five kinds; these are direct bridge exports,
|
||||
# not nonexistent sockets on DistributedCollector.
|
||||
data = compile_request('exact')
|
||||
binding = next(b for b in data['media']['outputs'] if b['kind'] == 'AUDIO')
|
||||
binding.update(socket=socket, kind=kind)
|
||||
inputs = data['workflow'][binding['node_id']]['inputs']
|
||||
inputs.pop('audio')
|
||||
inputs.update({socket: ['source:1', 1], 'binding_json': json.dumps(binding)})
|
||||
handler._validate_request(data)
|
||||
inputs['transfer_format'] = 'jpeg'
|
||||
with pytest.raises(ValueError):
|
||||
handler._validate_request(data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fmt', FORMATS)
|
||||
def test_master_passthrough_stays_native_for_every_format(module, fmt):
|
||||
def test_master_passthrough_stays_native_for_every_format(collector_module, fmt):
|
||||
media = native_media()
|
||||
actual = collect(module, **media, transfer_format=[fmt], transfer_quality=[73])
|
||||
for values, original in zip(actual, media.values()):
|
||||
assert values is original
|
||||
actual = collector_module.DistributedCollectorNode().run(
|
||||
images=[media['images']], audio=[media['audio']], pass_through=[True],
|
||||
transfer_format=[fmt], transfer_quality=[73])
|
||||
assert actual[0] is media['images']
|
||||
assert actual[1] is media['audio']
|
||||
assert_exact(actual[0], media['images'])
|
||||
|
||||
|
||||
@pytest.mark.parametrize('quality', [True, 0, 101, '90', [90,91]])
|
||||
def test_node_rejects_bad_quality_even_without_remote_participants(module, quality):
|
||||
with pytest.raises(ValueError):
|
||||
collect(module, texts=['hello'], transfer_quality=quality)
|
||||
def test_collector_widget_contract_keeps_saved_defaults_and_appends_exact(collector_module):
|
||||
optional = collector_module.DistributedCollectorNode.INPUT_TYPES()['optional']
|
||||
assert list(optional) == ['images', 'audio', 'transfer_format', 'transfer_quality']
|
||||
assert tuple(optional['transfer_format'][0]) == FORMATS
|
||||
assert optional['transfer_format'][1]['default'] == 'legacy_png'
|
||||
assert optional['transfer_quality'][1]['default'] == 90
|
||||
|
||||
@@ -24,7 +24,7 @@ compiler = importlib.import_module(f"{PACKAGE}.api.orchestration.serverless_medi
|
||||
legacy = importlib.import_module(f"{PACKAGE}.api.orchestration.serverless_prompt")
|
||||
contract = importlib.import_module(f"{PACKAGE}.serverless_bridge.contract")
|
||||
compile_media_prompt = compiler.compile_media_prompt
|
||||
SOCKETS = {"images": "IMAGE", "audio": "AUDIO", "masks": "MASK", "texts": "STRING", "video": "VIDEO"}
|
||||
SOCKETS = {"images": "IMAGE", "audio": "AUDIO"}
|
||||
|
||||
|
||||
def _node(class_type, **inputs):
|
||||
@@ -33,7 +33,7 @@ def _node(class_type, **inputs):
|
||||
|
||||
def _collector(**sockets):
|
||||
return _node(
|
||||
"DistributedMediaCollector", multi_job_id="job/42", worker_id="worker_0", **sockets
|
||||
"DistributedCollector", multi_job_id="job/42", worker_id="worker_0", **sockets
|
||||
)
|
||||
|
||||
|
||||
@@ -72,26 +72,26 @@ def test_one_export_for_each_typed_socket_with_exact_canonical_binding(socket, k
|
||||
"request_id": "request:1",
|
||||
"binding_json": json.dumps(binding),
|
||||
socket: ["source", 0],
|
||||
**({"transfer_format": "png", "transfer_quality": 90} if kind == "IMAGE" else {}),
|
||||
},
|
||||
}
|
||||
assert set(graph) == {"source", binding["node_id"]}
|
||||
|
||||
|
||||
def test_mixed_collectors_all_five_kinds_have_independent_dispatch_targets():
|
||||
def test_multiple_collectors_have_independent_image_audio_targets():
|
||||
prompt = {
|
||||
"shared": _node("MultiMediaSource"),
|
||||
"collector:z": _collector(audio=["shared", 1], video=["shared", 4]),
|
||||
"collector:a": _collector(images=["shared", 0], masks=["shared", 2], texts=["shared", 3]),
|
||||
"collector:z": _collector(audio=["shared", 1]),
|
||||
"collector:a": _collector(images=["shared", 0], audio=["shared", 1]),
|
||||
}
|
||||
prompt["collector:z"]["inputs"].update(multi_job_id="job:z", worker_id="participant:z")
|
||||
graph, bindings = _run(prompt)
|
||||
assert [(b["collector_id"], b["socket"]) for b in bindings] == [
|
||||
("collector:a", "images"), ("collector:a", "masks"), ("collector:a", "texts"),
|
||||
("collector:z", "audio"), ("collector:z", "video"),
|
||||
("collector:a", "images"), ("collector:a", "audio"), ("collector:z", "audio"),
|
||||
]
|
||||
assert len({b["binding_id"] for b in bindings}) == 5
|
||||
assert len({b["binding_id"] for b in bindings}) == 3
|
||||
targets = {b["node_id"] for b in bindings}
|
||||
assert len(targets) == 5
|
||||
assert len(targets) == 3
|
||||
assert set(graph) == {"shared"} | targets
|
||||
for binding in bindings:
|
||||
export = graph[binding["node_id"]]
|
||||
@@ -119,12 +119,15 @@ def test_prunes_descendants_and_unrelated_outputs_preserving_required_output_uti
|
||||
assert not {"load_balance", "pass_through"} & graph[bindings[0]["node_id"]]["inputs"].keys()
|
||||
|
||||
|
||||
def test_stock_primitive_string_preserves_empty_and_link_looking_text():
|
||||
def test_upstream_primitive_string_preserves_empty_and_link_looking_text():
|
||||
for text in ("", '["source:1", 0]', "hello\nworld"):
|
||||
prompt = {"text": _node("PrimitiveString", value=text), "collector": _collector(texts=["text", 0])}
|
||||
prompt = {"text": _node("PrimitiveString", value=text),
|
||||
"source": _node("TextToImage", text=["text", 0]),
|
||||
"collector": _collector(images=["source", 0])}
|
||||
graph, bindings = _run(prompt)
|
||||
assert graph["text"] == prompt["text"]
|
||||
assert graph[bindings[0]["node_id"]]["inputs"]["texts"] == ["text", 0]
|
||||
assert graph["source"]["inputs"]["text"] == ["text", 0]
|
||||
assert graph[bindings[0]["node_id"]]["inputs"]["images"] == ["source", 0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("class_type,inputs", [
|
||||
@@ -176,11 +179,13 @@ def test_distributed_text_value_can_be_wrapped_by_primitive_string():
|
||||
prompt = {
|
||||
"value": _node("DistributedModelName", text="model.safetensors"),
|
||||
"text": _node("PrimitiveString", value=["value", 0]),
|
||||
"collector": _collector(texts=["text", 0]),
|
||||
"source": _node("TextToImage", text=["text", 0]),
|
||||
"collector": _collector(images=["source", 0]),
|
||||
}
|
||||
graph, bindings = _run(prompt)
|
||||
assert graph["text"]["inputs"]["value"] == "model.safetensors"
|
||||
assert graph[bindings[0]["node_id"]]["inputs"]["texts"] == ["text", 0]
|
||||
assert graph["source"]["inputs"]["text"] == ["text", 0]
|
||||
assert graph[bindings[0]["node_id"]]["inputs"]["images"] == ["source", 0]
|
||||
assert "value" not in graph
|
||||
|
||||
|
||||
@@ -321,7 +326,7 @@ def test_rejects_invalid_links_everywhere_before_pruning(location, link):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("class_type", [
|
||||
"DistributedCollector", "DistributedEmptyImage", "UltimateSDUpscaleDistributed",
|
||||
"DistributedMediaCollector", "DistributedEmptyImage", "UltimateSDUpscaleDistributed",
|
||||
"DistributedImageListStart", "DistributedImageListEnd", "DistributedWorkerSink",
|
||||
"ImageBatchDivider", "AudioBatchDivider", "CustomDistributedNode", "custombatchdivider",
|
||||
])
|
||||
@@ -353,9 +358,9 @@ def test_rejects_collector_chains_across_different_media(indirect):
|
||||
prompt = _prompt()
|
||||
source = "collector:1"
|
||||
if indirect:
|
||||
prompt["processing"] = _node("ImageToVideo", images=[source, 0])
|
||||
prompt["processing"] = _node("ImageToAudio", images=[source, 0])
|
||||
source = "processing"
|
||||
prompt["collector:2"] = _collector(video=[source, 0])
|
||||
prompt["collector:2"] = _collector(audio=[source, 0])
|
||||
with pytest.raises(ValueError, match="collector-to-collector"):
|
||||
_run(prompt)
|
||||
|
||||
@@ -413,7 +418,7 @@ def guarded_import(name, *args, **kwargs):
|
||||
return original_import(name, *args, **kwargs)
|
||||
builtins.__import__ = guarded_import
|
||||
module = importlib.import_module(name + ".api.orchestration.serverless_media_prompt")
|
||||
prompt = {"s": {"class_type": "Source", "inputs": {}}, "c": {"class_type": "DistributedMediaCollector", "inputs": {"texts": ["s", 0], "multi_job_id": "job", "worker_id": "worker"}}}
|
||||
prompt = {"s": {"class_type": "Source", "inputs": {}}, "c": {"class_type": "DistributedCollector", "inputs": {"images": ["s", 0], "multi_job_id": "job", "worker_id": "worker"}}}
|
||||
graph, bindings = module.compile_media_prompt(prompt, "request")
|
||||
assert graph[bindings[0]["node_id"]]["class_type"] == "MediaExportV1"
|
||||
'''
|
||||
|
||||
@@ -133,6 +133,7 @@ def test_quality_to_crf_mapping():
|
||||
|
||||
def test_normalize_format_rejects_unknown_values():
|
||||
assert tc.normalize_format(" H264 ") == "h264"
|
||||
assert tc.normalize_format(" EXACT ") == "exact"
|
||||
assert tc.normalize_format(None) == tc.DEFAULT_TRANSFER_FORMAT
|
||||
with pytest.raises(tc.TransferCodecError):
|
||||
tc.normalize_format("gif")
|
||||
@@ -158,6 +159,22 @@ def test_decode_chunk_validates_envelope_against_payload():
|
||||
tc.decode_chunk({**video_env, "count": 5}, video["parts"])
|
||||
|
||||
|
||||
def test_exact_is_appended_without_changing_saved_format_defaults():
|
||||
assert tc.TRANSFER_FORMATS == (
|
||||
"legacy_png", "png", "ffv1", "jpeg", "webp", "h264", "h265", "av1", "exact",
|
||||
)
|
||||
assert tc.DEFAULT_TRANSFER_FORMAT == "legacy_png"
|
||||
assert tc.EXACT_FORMAT not in tc.IMAGE_FORMATS + tc.VIDEO_FORMATS
|
||||
|
||||
|
||||
def test_exact_does_not_enter_uint8_chunk_codec():
|
||||
frames = tc.tensor_batch_to_uint8(_smooth_frames(count=1))
|
||||
with pytest.raises(tc.TransferCodecError, match="exact"):
|
||||
list(tc.encode_chunks(frames, "exact", 90, 10**9))
|
||||
with pytest.raises(tc.TransferCodecError, match="exact"):
|
||||
tc.decode_chunk({"format": "exact", "count": 1, "width": 91, "height": 67}, [b"raw tensor"])
|
||||
|
||||
|
||||
def test_legacy_format_is_not_chunk_encoded():
|
||||
frames = tc.tensor_batch_to_uint8(_smooth_frames(count=1))
|
||||
with pytest.raises(tc.TransferCodecError):
|
||||
|
||||
@@ -69,8 +69,8 @@ def v3(monkeypatch):
|
||||
module(f"{PKG}.nodes.utilities")
|
||||
module(f"{PKG}.nodes.distributed_upscale", UltimateSDUpscaleDistributed=object)
|
||||
module(f"{PKG}.nodes.collector", DistributedCollectorNode=CollectorRuntime)
|
||||
module(f"{PKG}.nodes.image_list", DistributedImageListStart=object, DistributedImageListEnd=object)
|
||||
module(f"{PKG}.nodes.media_collector", DistributedMediaCollector=object)
|
||||
module(f"{PKG}.nodes.image_list", DistributedImageListStart=object, DistributedImageListEnd=object,
|
||||
DistributedImageListGetValue=object)
|
||||
module("comfy", __path__=[])
|
||||
module("comfy.samplers", KSampler=types.SimpleNamespace(SAMPLERS=["euler"], SCHEDULERS=["normal"]))
|
||||
module("comfy_api", __path__=[])
|
||||
@@ -98,6 +98,7 @@ def test_registered_collector_schema_exposes_optional_transfer_inputs(v3):
|
||||
fmt = inputs["transfer_format"]
|
||||
assert fmt.kind == "Combo"
|
||||
assert fmt.kwargs["options"] == list(codec.TRANSFER_FORMATS)
|
||||
assert fmt.kwargs["options"][-1] == "exact"
|
||||
assert fmt.kwargs["default"] == "legacy_png"
|
||||
assert fmt.kwargs["optional"] is True
|
||||
quality = inputs["transfer_quality"]
|
||||
@@ -107,13 +108,14 @@ def test_registered_collector_schema_exposes_optional_transfer_inputs(v3):
|
||||
assert schema.is_input_list is True
|
||||
|
||||
|
||||
def test_registered_collector_forwards_transfer_options(v3):
|
||||
@pytest.mark.parametrize("fmt", ["av1", "exact"])
|
||||
def test_registered_collector_forwards_transfer_options(v3, fmt):
|
||||
mod, _, calls = v3
|
||||
|
||||
mod.DistributedCollector.execute(images=["img"], audio=[None], transfer_format=["av1"], transfer_quality=[70],
|
||||
mod.DistributedCollector.execute(images=["img"], audio=[None], transfer_format=[fmt], transfer_quality=[70],
|
||||
multi_job_id=["job"], is_worker=[True])
|
||||
|
||||
assert calls[-1]["transfer_format"] == ["av1"]
|
||||
assert calls[-1]["transfer_format"] == [fmt]
|
||||
assert calls[-1]["transfer_quality"] == [70]
|
||||
assert calls[-1]["multi_job_id"] == ["job"]
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ def test_image_list_network_runtime():
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + '\n' + result.stderr
|
||||
assert 'IMAGE_LIST_NETWORK_OK' in result.stdout
|
||||
assert 'IMAGE_LIST_VALUES_OK' in result.stdout
|
||||
assert 'IMAGE_LIST_CANCEL_OK' in result.stdout
|
||||
assert 'IMAGE_LIST_MASTER_OK' in result.stdout
|
||||
assert 'IMAGE_LIST_PREFLIGHT_OK' in result.stdout
|
||||
|
||||
Reference in New Issue
Block a user