From a1a21ab67282690cc9655a8a01ab04ffb7a45131 Mon Sep 17 00:00:00 2001 From: silveroxides Date: Tue, 6 Oct 2026 09:51:40 +0200 Subject: [PATCH] Place image parts before video parts in multimodal payloads --- gemini_nodes.py | 26 +++++++++++++------------- interactions_nodes.py | 24 ++++++++++++------------ tests/test_gemini_nodes.py | 7 ++++--- tests/test_interactions_nodes.py | 19 +++++++++++++++++++ 4 files changed, 48 insertions(+), 28 deletions(-) diff --git a/gemini_nodes.py b/gemini_nodes.py index 274c3e3..a4f595b 100644 --- a/gemini_nodes.py +++ b/gemini_nodes.py @@ -1012,7 +1012,7 @@ class SSL_GeminiTextPrompt(IO.ComfyNode): print(f"[ERROR] Gemini client initialization failed: {str(e)}") return IO.NodeOutput(f"Gemini client initialization failed: {str(e)}", cls.generate_empty_image(), actual_seed if actual_seed is not None else 0, "", "") - # Prepare contents (video + images + prompt) + # Prepare contents (images + video + prompt) image_frames, image_batch_counts = cls._ordered_image_frames(image_inputs) context_image_hashes = [] media_parts = [] @@ -1022,18 +1022,6 @@ class SSL_GeminiTextPrompt(IO.ComfyNode): + len(padded_system_instruction.encode("utf-8")) ) - if video_bytes is not None: - if model in cls.MEDIA_RES_MODELS and media_resolution is not None and media_resolution != "unspecified": - video_part = types.Part.from_bytes( - data=video_bytes, - mime_type=video_mime_type, - media_resolution=f"MEDIA_RESOLUTION_{media_resolution.upper()}", - ) - else: - video_part = types.Part.from_bytes(data=video_bytes, mime_type=video_mime_type) - video_part.video_metadata = types.VideoMetadata(fps=video_fps) - media_parts.append(video_part) - if image_frames: try: img_parts = [] @@ -1070,6 +1058,18 @@ class SSL_GeminiTextPrompt(IO.ComfyNode): except Exception as e: print(f"[ERROR] Error processing input image: {str(e)}") return IO.NodeOutput(f"Error processing input image: {str(e)}", cls.generate_empty_image(), actual_seed if actual_seed is not None else 0, "", "") + + if video_bytes is not None: + if model in cls.MEDIA_RES_MODELS and media_resolution is not None and media_resolution != "unspecified": + video_part = types.Part.from_bytes( + data=video_bytes, + mime_type=video_mime_type, + media_resolution=f"MEDIA_RESOLUTION_{media_resolution.upper()}", + ) + else: + video_part = types.Part.from_bytes(data=video_bytes, mime_type=video_mime_type) + video_part.video_metadata = types.VideoMetadata(fps=video_fps) + media_parts.append(video_part) if media_parts: contents = [ types.UserContent( diff --git a/interactions_nodes.py b/interactions_nodes.py index 5876716..0165a19 100644 --- a/interactions_nodes.py +++ b/interactions_nodes.py @@ -841,17 +841,6 @@ class SSL_GeminiInteractionsTextPrompt(IO.ComfyNode): image_frames, image_batch_counts = cls._ordered_image_frames(image_inputs) media_items = [] - if video_bytes is not None: - video_stream = BytesIO(video_bytes) - v_item = { - "type": "video", - "data": video_stream, - "mime_type": video_mime_type, - } - if media_resolution and media_resolution != "unspecified": - v_item["resolution"] = media_resolution.lower() - media_items.append(v_item) - if image_frames: try: for _, _, image_frame in image_frames: @@ -875,12 +864,23 @@ class SSL_GeminiInteractionsTextPrompt(IO.ComfyNode): ) print( f"[INFO] Prepared Gemini image inputs: {batch_summary}, " - f"total={len(media_items) - (1 if video_bytes is not None else 0)}" + f"total={len(media_items)}" ) except Exception as e: print(f"[ERROR] Error processing input image: {str(e)}") return IO.NodeOutput(f"Error processing input image: {str(e)}", cls.generate_empty_image(), actual_seed if actual_seed is not None else 0, "", "", "") + if video_bytes is not None: + video_stream = BytesIO(video_bytes) + v_item = { + "type": "video", + "data": video_stream, + "mime_type": video_mime_type, + } + if media_resolution and media_resolution != "unspecified": + v_item["resolution"] = media_resolution.lower() + media_items.append(v_item) + text_item = {"type": "text", "text": padded_prompt} if media_items: input_contents = [*media_items, text_item] diff --git a/tests/test_gemini_nodes.py b/tests/test_gemini_nodes.py index e63a89c..5fb2573 100644 --- a/tests/test_gemini_nodes.py +++ b/tests/test_gemini_nodes.py @@ -531,9 +531,10 @@ def test_context_cache_reuses_full_input_after_local_fingerprint_miss(monkeypatc create_config = captured["creates"][0]["config"] assert create_config.system_instruction == "test system instruction" assert create_config.ttl == "900s" - assert create_config.contents[0].parts[0].inline_data.data == b"normalized-video" - assert create_config.contents[0].parts[0].inline_data.mime_type == "video/mp4" - assert create_config.contents[0].parts[0].video_metadata.fps == 2 + assert create_config.contents[0].parts[0].inline_data.mime_type == "image/png" + assert create_config.contents[0].parts[1].inline_data.data == b"normalized-video" + assert create_config.contents[0].parts[1].inline_data.mime_type == "video/mp4" + assert create_config.contents[0].parts[1].video_metadata.fps == 2 assert create_config.contents[0].parts[-1].text == "test prompt" assert len(create_config.contents[0].parts) == 3 assert len(captured["generations"]) == 2 diff --git a/tests/test_interactions_nodes.py b/tests/test_interactions_nodes.py index 094ceb2..3d0c80c 100644 --- a/tests/test_interactions_nodes.py +++ b/tests/test_interactions_nodes.py @@ -244,6 +244,25 @@ def test_multimodal_video_input(mock_interactions_client): assert base64.b64decode(validated_vid.data) == video_data +def test_multimodal_image_and_video_ordering(mock_interactions_client): + mock_interactions_client.responses = [{"text": "Found image and video", "id": "int_both_1"}] + video_data = b"fake video bytes" + image_tensor = torch.zeros(1, 16, 16, 3) + args = _execute_kwargs( + video={"video": FakeVideo(video_data), "fps": 2, "pad_at_start": False, "duration_aware_padding": False}, + image_inputs={"image_1": image_tensor}, + prompt="Describe image and video", + ) + output = interactions_nodes.SSL_GeminiInteractionsTextPrompt.execute(**args) + assert output[0] == "Found image and video" + call = mock_interactions_client.calls[0] + items = call["input"] + assert len(items) == 3 + assert items[0]["type"] == "image" + assert items[1]["type"] == "video" + assert items[2]["type"] == "text" + + def test_structured_output_with_schema(mock_interactions_client): schema = { "type": "object",