From 9f4f51aaef56e63ee33c148a4eb956ce1d10c48e Mon Sep 17 00:00:00 2001 From: silveroxides Date: Tue, 1 Sep 2026 08:56:29 +0200 Subject: [PATCH] feat(video): add configurable start padding --- README.md | 9 ++++- gemini_nodes.py | 80 +++++++++++++++++++++++++++++++++--- tests/test_gemini_nodes.py | 83 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 162 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index f5de778..c9f9f68 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Use the `Expanded Gemini Text/Image` node for text generation: - Enter your prompt text in `prompt`. - Adjust generation parameters (temperature, top_p, top_k, etc.). - If needed, connect images through `image_inputs`. For video, pass a native ComfyUI video through `Configure Gemini Video Input`, then connect its output to `video`. -- Configured video is normalized to MP4/H.264 and sent with embedded audio at the selected sampling FPS. The normalized video must be smaller than 100 MB. +- Configured video is normalized to MP4/H.264 and sent with embedded audio at the selected sampling FPS. `pad_at_start` optionally prepends black video and silence. `duration_aware_padding` derives that padding from the video's fractional-second duration and rounds it down to native video frames for comparison testing. The normalized video must be smaller than 100 MB. - Enable `use_cache` on the configuration node to reuse unchanged prompt, system instruction, image, and video context through Gemini after the local result cache misses. ### 3. Proxy Settings @@ -77,6 +77,13 @@ If you are in China or other regions requiring a proxy: - `cache_ttl_minutes`: Number of minutes Gemini retains cached context. - `cache_seed`: Generation seed used while `use_cache` is enabled. +### Video Input Configuration Node + +- `video`: Native ComfyUI video input. +- `fps`: Gemini sampling rate from 1 through 24 FPS. +- `pad_at_start`: Prepend matching-resolution black video and silence for half one sampling interval before encoding. +- `duration_aware_padding`: With start padding enabled, use the video's fractional-second duration rounded down to native video frames instead of the standard padding duration. + ### Text Generation Node #### Required Parameters diff --git a/gemini_nodes.py b/gemini_nodes.py index 043cbaf..9cac497 100644 --- a/gemini_nodes.py +++ b/gemini_nodes.py @@ -8,7 +8,7 @@ import cv2 from PIL import Image from io import BytesIO import folder_paths # type: ignore[reportMissingImports] -from comfy_api.latest import ComfyExtension, UI, IO, Types # type: ignore[reportMissingImports] +from comfy_api.latest import ComfyExtension, UI, IO, Types, VideoFromComponents # type: ignore[reportMissingImports] from google import genai from google.genai import errors, types import time @@ -157,13 +157,20 @@ class SSL_GeminiVideoConfig(IO.ComfyNode): inputs=[ IO.Video.Input("video"), IO.Int.Input("fps", default=1, min=1, max=24, step=1, tooltip="Frames per second sampled by Gemini for video understanding."), + IO.Boolean.Input("pad_at_start", default=False, tooltip="Prepend black video and silence for half one Gemini sampling interval."), + IO.Boolean.Input("duration_aware_padding", default=False, tooltip="Use the video's fractional-second duration, rounded down to native video frames, as the start padding duration."), ], outputs=[cls.GeminiVideoConfig.Output("video_config")], ) @classmethod - def execute(cls, video: IO.Video.Type, fps: int) -> IO.NodeOutput: - return IO.NodeOutput({"video": video, "fps": int(fps)}) + def execute(cls, video: IO.Video.Type, fps: int, pad_at_start: bool = False, duration_aware_padding: bool = False) -> IO.NodeOutput: + return IO.NodeOutput({ + "video": video, + "fps": int(fps), + "pad_at_start": bool(pad_at_start), + "duration_aware_padding": bool(duration_aware_padding), + }) class SSL_GeminiTextPrompt(IO.ComfyNode): @@ -347,12 +354,66 @@ class SSL_GeminiTextPrompt(IO.ComfyNode): return frames, batch_counts @classmethod - def _serialize_video(cls, video: IO.Video.Type | None): + def _serialize_video(cls, video: IO.Video.Type | None, fps: int | None = None, + pad_at_start: bool = False, duration_aware_padding: bool = False): if video is None: return None, None + video_to_save = video + if pad_at_start: + if fps is None: + raise ValueError("Gemini video FPS is required when padding at start.") + components = video.get_components() + native_frame_rate = components.frame_rate + source_frame_rate = float(native_frame_rate) + source_images = components.images + source_frame_count = source_images.shape[0] + if duration_aware_padding: + source_duration = video.get_duration() + target_frame_rate = native_frame_rate + padding_frame_count = int((source_duration % 1.0) * source_frame_rate) + resampled_images = source_images + else: + source_duration = source_frame_count / source_frame_rate + target_frame_rate = fps * 2 + padding_frame_count = 1 + target_frame_count = max(1, round(source_duration * target_frame_rate)) + source_indices = ( + torch.arange(target_frame_count, device=source_images.device, dtype=torch.float64) + * source_frame_rate + / target_frame_rate + ).floor().long().clamp(max=source_frame_count - 1) + resampled_images = source_images.index_select(0, source_indices) + black_frames = torch.zeros_like(resampled_images[:1]).expand(padding_frame_count, -1, -1, -1) + padded_images = torch.cat((black_frames, resampled_images), dim=0) + + padded_audio = components.audio + if padded_audio: + sample_rate = int(padded_audio["sample_rate"]) + waveform = padded_audio["waveform"] + silence_samples = round(sample_rate * padding_frame_count / target_frame_rate) + silence = torch.zeros( + (*waveform.shape[:-1], silence_samples), + dtype=waveform.dtype, + device=waveform.device, + ) + padded_audio = { + **padded_audio, + "waveform": torch.cat((silence, waveform), dim=-1), + } + + video_to_save = VideoFromComponents( + Types.VideoComponents( + images=padded_images, + audio=padded_audio, + frame_rate=target_frame_rate, + ), + bit_depth=video.get_bit_depth(), + color_space=video.get_color_space(), + ) + buffer = BytesIO() - video.save_to( + video_to_save.save_to( buffer, format=Types.VideoContainer.MP4, codec=Types.VideoCodec.H264, @@ -723,8 +784,15 @@ class SSL_GeminiTextPrompt(IO.ComfyNode): cache_seed = int(config.get("cache_seed", 0)) video_input = video.get("video") if video is not None else None video_fps = int(video.get("fps", 1)) if video is not None else None + video_pad_at_start = bool(video.get("pad_at_start", False)) if video is not None else False + video_duration_aware_padding = bool(video.get("duration_aware_padding", False)) if video is not None else False try: - video_bytes, video_mime_type = cls._serialize_video(video_input) + video_bytes, video_mime_type = cls._serialize_video( + video_input, + video_fps, + video_pad_at_start, + video_duration_aware_padding, + ) except Exception as e: print(f"[ERROR] Error processing input video: {e}") output_seed = cache_seed if use_cache and use_seed else seed if use_seed else 0 diff --git a/tests/test_gemini_nodes.py b/tests/test_gemini_nodes.py index 851e768..ab21eed 100644 --- a/tests/test_gemini_nodes.py +++ b/tests/test_gemini_nodes.py @@ -84,14 +84,91 @@ def test_image_autogrow_is_last_input(): def test_video_config_exposes_native_video_and_integer_fps(): schema = gemini_nodes.SSL_GeminiVideoConfig.define_schema() fps = next(input_ for input_ in schema.inputs if input_.id == "fps") + pad_at_start = next(input_ for input_ in schema.inputs if input_.id == "pad_at_start") + duration_aware_padding = next(input_ for input_ in schema.inputs if input_.id == "duration_aware_padding") - assert [input_.id for input_ in schema.inputs] == ["video", "fps"] + assert [input_.id for input_ in schema.inputs] == ["video", "fps", "pad_at_start", "duration_aware_padding"] assert fps.default == 1 assert fps.min == 1 assert fps.max == 24 + assert pad_at_start.default is False + assert duration_aware_padding.default is False native_video = object() - output = gemini_nodes.SSL_GeminiVideoConfig.execute(native_video, 2) - assert output[0] == {"video": native_video, "fps": 2} + output = gemini_nodes.SSL_GeminiVideoConfig.execute(native_video, 2, True, True) + assert output[0] == { + "video": native_video, + "fps": 2, + "pad_at_start": True, + "duration_aware_padding": True, + } + + +def test_video_start_padding_matches_resolution_and_half_sampling_interval(monkeypatch): + images = torch.ones((4, 3, 5, 3)) + waveform = torch.ones((1, 2, 8000)) + video = SimpleNamespace( + get_components=lambda: gemini_nodes.Types.VideoComponents( + images=images, + audio={"waveform": waveform, "sample_rate": 8000}, + frame_rate=4, + ), + get_bit_depth=lambda: 8, + get_color_space=lambda: "sRGB", + ) + captured = {} + + class CapturingVideo: + def __init__(self, components, bit_depth, color_space): + captured.update(components=components, bit_depth=bit_depth, color_space=color_space) + + def save_to(self, buffer, format, codec): + buffer.write(b"padded-video") + + monkeypatch.setattr(gemini_nodes, "VideoFromComponents", CapturingVideo) + + video_bytes, mime_type = gemini_nodes.SSL_GeminiTextPrompt._serialize_video(video, fps=2, pad_at_start=True) + + assert video_bytes == b"padded-video" + assert mime_type == "video/mp4" + assert captured["components"].frame_rate == 4 + assert captured["components"].images.shape == (5, 3, 5, 3) + assert torch.count_nonzero(captured["components"].images[0]) == 0 + assert torch.equal(captured["components"].images[1], images[0]) + assert captured["components"].audio["waveform"].shape[-1] == 10000 + assert torch.count_nonzero(captured["components"].audio["waveform"][..., :2000]) == 0 + + +def test_duration_aware_padding_preserves_native_fps_and_rounds_down_to_frames(monkeypatch): + images = torch.ones((422, 1, 1, 3)) + video = SimpleNamespace( + get_components=lambda: gemini_nodes.Types.VideoComponents(images=images, frame_rate=40), + get_duration=lambda: 10.559, + get_bit_depth=lambda: 8, + get_color_space=lambda: "sRGB", + ) + captured = {} + + class CapturingVideo: + def __init__(self, components, bit_depth, color_space): + captured["components"] = components + + def save_to(self, buffer, format, codec): + buffer.write(b"duration-aware-video") + + monkeypatch.setattr(gemini_nodes, "VideoFromComponents", CapturingVideo) + + gemini_nodes.SSL_GeminiTextPrompt._serialize_video( + video, + fps=4, + pad_at_start=True, + duration_aware_padding=True, + ) + + components = captured["components"] + assert components.frame_rate == 40 + assert components.images.shape[0] == 22 + images.shape[0] + assert torch.count_nonzero(components.images[:22]) == 0 + assert torch.equal(components.images[22], images[0]) def test_context_cache_reuses_full_input_after_local_fingerprint_miss(monkeypatch):