feat: add flux 3 video generation nodes
Wrap the new flux-3-video endpoint (t2v, i2v, v2v modes) in three generation nodes plus two workflow utilities: - BaseFlux3Video reuses BaseFlux post/poll, overrides process_result to download the signed MP4 and return ComfyUI's native VIDEO type, and returns a 1-frame black video on every failure path. Polling ceiling is 240 attempts (~20 min): a real v2v+fhd task was still generating at 11 minutes. - Flux3Keyframes combines up to 10 image sockets into the keyframes string; timing 'even' sends a plain list, 'custom' sends [seconds, image] pairs (start pinned at 0, end_time = clip length). - VideoToBase64 converts a ComfyUI VIDEO to a base64 MP4 string for v2v's start_video. Nodes register only when comfy_api is importable so the package keeps loading on older ComfyUI versions.
This commit is contained in:
Çağan Kiraz
@@ -5,6 +5,7 @@ node_list = [
|
||||
"api_node",
|
||||
"finetune",
|
||||
"flux_tools",
|
||||
"flux3_video",
|
||||
"config_node",
|
||||
"utils",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
import io
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
|
||||
from .base import REQUEST_TIMEOUT, BaseFlux
|
||||
|
||||
try:
|
||||
from comfy_api.input_impl import VideoFromFile
|
||||
except ImportError:
|
||||
VideoFromFile = None
|
||||
print("[BFL] comfy_api.input_impl not found — Flux 3 Video nodes disabled (requires ComfyUI >= 0.3.30)")
|
||||
|
||||
VIDEO_DURATIONS = ["auto"] + [str(i) for i in range(5, 21)]
|
||||
VIDEO_ASPECT_RATIOS = ["auto", "21:9", "2:1", "16:9", "4:3", "1:1", "3:4", "9:16"]
|
||||
|
||||
|
||||
class BaseFlux3Video(BaseFlux):
|
||||
CATEGORY = "BFL/Flux3"
|
||||
RETURN_TYPES = ("VIDEO",)
|
||||
RETURN_NAMES = ("video",)
|
||||
FUNCTION = "generate_video"
|
||||
|
||||
# Video generation outlasts the 40-attempt image ceiling; 5s interval -> ~20 min.
|
||||
# A real v2v+fhd task was still Generating at 11 min, so 120 attempts proved too low.
|
||||
VIDEO_MAX_ATTEMPTS = 240
|
||||
|
||||
@classmethod
|
||||
def common_input_types(cls):
|
||||
return {
|
||||
"prompt": ("STRING", {"default": "", "multiline": True}),
|
||||
"resolution": (["hd", "fhd"], {"default": "hd"}),
|
||||
"duration": (VIDEO_DURATIONS, {"default": "auto"}),
|
||||
"aspect_ratio": (VIDEO_ASPECT_RATIOS, {"default": "auto"}),
|
||||
"generate_audio": ("BOOLEAN", {"default": True}),
|
||||
"safety_tolerance": ("INT", {"default": 2, "min": 0, "max": 4}),
|
||||
"draft": ("BOOLEAN", {"default": False}),
|
||||
}
|
||||
|
||||
def build_arguments(
|
||||
self, mode, prompt, resolution, duration, aspect_ratio, generate_audio, safety_tolerance, draft
|
||||
):
|
||||
arguments = {"mode": mode, "prompt": prompt}
|
||||
if resolution != "hd":
|
||||
arguments["resolution"] = resolution
|
||||
if duration != "auto":
|
||||
arguments["duration"] = int(duration)
|
||||
if aspect_ratio != "auto":
|
||||
arguments["aspect_ratio"] = aspect_ratio
|
||||
if not generate_audio:
|
||||
arguments["generate_audio"] = False
|
||||
if safety_tolerance != 2:
|
||||
arguments["safety_tolerance"] = safety_tolerance
|
||||
if draft:
|
||||
arguments["draft"] = True
|
||||
return arguments
|
||||
|
||||
def process_result(self, result, output_format="jpeg"):
|
||||
try:
|
||||
sample_url = result["result"]["sample"]
|
||||
video_response = requests.get(sample_url, timeout=REQUEST_TIMEOUT)
|
||||
video_response.raise_for_status()
|
||||
size_mb = len(video_response.content) / (1024 * 1024)
|
||||
print(f"[BFL] Video downloaded ({size_mb:.1f} MB)")
|
||||
return (VideoFromFile(io.BytesIO(video_response.content)),)
|
||||
except KeyError as e:
|
||||
print(f"[BFL] KeyError: Missing expected key {e}")
|
||||
return self.create_blank_image()
|
||||
except Exception as e:
|
||||
print(f"[BFL] Error processing video result: {str(e)}")
|
||||
return self.create_blank_image()
|
||||
|
||||
def create_blank_image(self):
|
||||
"""Return a 1-frame black 512x512 VIDEO instead of an IMAGE tensor.
|
||||
|
||||
Keeps the BaseFlux method name because every failure path in
|
||||
BaseFlux.get_result / post_request calls create_blank_image();
|
||||
a VIDEO-returning node must never emit an IMAGE tensor.
|
||||
"""
|
||||
import av # ships with ComfyUI; only needed on failure paths
|
||||
|
||||
buffer = io.BytesIO()
|
||||
container = av.open(buffer, mode="w", format="mp4")
|
||||
stream = container.add_stream("h264", rate=1)
|
||||
stream.width = 512
|
||||
stream.height = 512
|
||||
stream.pix_fmt = "yuv420p"
|
||||
frame = av.VideoFrame.from_ndarray(np.zeros((512, 512, 3), dtype=np.uint8), format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
container.close()
|
||||
buffer.seek(0)
|
||||
return (VideoFromFile(buffer),)
|
||||
|
||||
def generate_video_request(self, arguments, config=None):
|
||||
try:
|
||||
task_id = self.post_request("flux-3-video", arguments, config)
|
||||
if task_id:
|
||||
print(f"Task ID '{task_id}'")
|
||||
return self.get_result(task_id, max_attempts=self.VIDEO_MAX_ATTEMPTS, config_override=config)
|
||||
return self.create_blank_image()
|
||||
except Exception as e:
|
||||
print(f"Error generating video: {str(e)}")
|
||||
return self.create_blank_image()
|
||||
|
||||
|
||||
class Flux3VideoT2V(BaseFlux3Video):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {"required": cls.common_input_types(), "optional": {"config": ("BFL_CONFIG",)}}
|
||||
|
||||
def generate_video(
|
||||
self, prompt, resolution, duration, aspect_ratio, generate_audio, safety_tolerance, draft, config=None
|
||||
):
|
||||
arguments = self.build_arguments(
|
||||
"t2v", prompt, resolution, duration, aspect_ratio, generate_audio, safety_tolerance, draft
|
||||
)
|
||||
return self.generate_video_request(arguments, config)
|
||||
|
||||
|
||||
class Flux3Keyframes:
|
||||
"""Combine up to 10 images into the keyframes string for Flux 3 Video I2V.
|
||||
|
||||
Empty sockets are skipped. With timing "even" the connected images are sent
|
||||
as a plain list: the first starts the video, the last ends it, the middles
|
||||
fall evenly in between (per the BFL API, 3 or more need a set duration on
|
||||
the video node). With timing "custom" every image is sent as a
|
||||
[seconds, image] pair — the API allows no mixing, so all frames get a time:
|
||||
start_image is pinned at 0, each middle at its time widget, and end_image
|
||||
at end_time, which with duration "auto" also sets the clip length ("the
|
||||
video runs to the last pair's second, rounded up").
|
||||
"""
|
||||
|
||||
CATEGORY = "BFL/Flux3"
|
||||
RETURN_TYPES = ("STRING",)
|
||||
RETURN_NAMES = ("keyframes",)
|
||||
FUNCTION = "combine"
|
||||
|
||||
MIDDLE_SLOTS = [(f"image_{i}", f"time_{i}") for i in range(2, 10)]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
image_socket = ("STRING", {"forceInput": True, "tooltip": "Base64 image (from Image to Base64) or image URL"})
|
||||
time_widget = (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 0.0,
|
||||
"min": 0.0,
|
||||
"max": 20.0,
|
||||
"step": 0.05,
|
||||
"tooltip": 'Second to pin the matching image at — only used when timing is "custom"',
|
||||
},
|
||||
)
|
||||
optional = {"start_image": image_socket}
|
||||
for image_name, time_name in cls.MIDDLE_SLOTS:
|
||||
optional[image_name] = image_socket
|
||||
optional[time_name] = time_widget
|
||||
optional["end_image"] = image_socket
|
||||
optional["end_time"] = (
|
||||
"FLOAT",
|
||||
{
|
||||
"default": 5.0,
|
||||
"min": 0.0,
|
||||
"max": 20.0,
|
||||
"step": 0.05,
|
||||
"tooltip": 'Second the end image lands at (timing "custom") — with duration "auto" '
|
||||
"this is the clip length",
|
||||
},
|
||||
)
|
||||
return {
|
||||
"required": {
|
||||
"timing": (
|
||||
["even", "custom"],
|
||||
{
|
||||
"default": "even",
|
||||
"tooltip": "even: images fall evenly across the clip; custom: start at 0, middles at "
|
||||
"their time widgets, end at end_time",
|
||||
},
|
||||
),
|
||||
},
|
||||
"optional": optional,
|
||||
}
|
||||
|
||||
def combine(self, timing="even", **inputs):
|
||||
frames = []
|
||||
if inputs.get("start_image"):
|
||||
frames.append([0.0, inputs["start_image"]])
|
||||
for image_name, time_name in self.MIDDLE_SLOTS:
|
||||
if inputs.get(image_name):
|
||||
frames.append([inputs.get(time_name, 0.0), inputs[image_name]])
|
||||
end_image = inputs.get("end_image")
|
||||
if end_image:
|
||||
frames.append([inputs.get("end_time", 0.0), end_image])
|
||||
if not frames:
|
||||
return ("",)
|
||||
if timing == "custom":
|
||||
frames.sort(key=lambda pair: pair[0])
|
||||
times = [pair[0] for pair in frames]
|
||||
if len(set(times)) != len(times):
|
||||
print(f"[BFL] Warning: duplicate keyframe times {times} — set a distinct time for each image")
|
||||
if end_image and frames[-1][1] is not end_image:
|
||||
print("[BFL] Warning: end_time is not the largest time — the end image will not end the video")
|
||||
return (json.dumps(frames),)
|
||||
return (json.dumps([image for _, image in frames]),)
|
||||
|
||||
|
||||
class Flux3VideoI2V(BaseFlux3Video):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
common = cls.common_input_types()
|
||||
return {
|
||||
"required": {
|
||||
"prompt": common.pop("prompt"),
|
||||
"keyframes": (
|
||||
"STRING",
|
||||
{
|
||||
"default": "",
|
||||
"multiline": True,
|
||||
"tooltip": "Connect a Flux 3 Keyframes node, or paste a single image URL/base64, "
|
||||
'or a JSON array of images or [seconds, image] pairs (up to 10): [[0, "..."], [4.5, "..."]]',
|
||||
},
|
||||
),
|
||||
**common,
|
||||
},
|
||||
"optional": {"config": ("BFL_CONFIG",)},
|
||||
}
|
||||
|
||||
def generate_video(
|
||||
self,
|
||||
prompt,
|
||||
keyframes,
|
||||
resolution,
|
||||
duration,
|
||||
aspect_ratio,
|
||||
generate_audio,
|
||||
safety_tolerance,
|
||||
draft,
|
||||
config=None,
|
||||
):
|
||||
arguments = self.build_arguments(
|
||||
"i2v", prompt, resolution, duration, aspect_ratio, generate_audio, safety_tolerance, draft
|
||||
)
|
||||
if not keyframes.strip():
|
||||
print("[BFL] i2v requires keyframes — connect a Flux 3 Keyframes node or an image string")
|
||||
return self.create_blank_image()
|
||||
try:
|
||||
arguments["keyframes"] = json.loads(keyframes)
|
||||
except ValueError:
|
||||
# Not JSON: treat as a single bare image (URL or base64) opening the clip, per the API docs
|
||||
arguments["keyframes"] = keyframes
|
||||
parsed = arguments["keyframes"]
|
||||
if (
|
||||
isinstance(parsed, list)
|
||||
and len(parsed) >= 3
|
||||
and all(isinstance(k, str) for k in parsed)
|
||||
and duration == "auto"
|
||||
):
|
||||
print("[BFL] Warning: 3+ plain keyframes need a set duration — BFL rejects duration=auto for these")
|
||||
return self.generate_video_request(arguments, config)
|
||||
|
||||
|
||||
class Flux3VideoV2V(BaseFlux3Video):
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
common = cls.common_input_types()
|
||||
return {
|
||||
"required": {"prompt": common.pop("prompt"), "start_video": ("STRING", {"default": ""}), **common},
|
||||
"optional": {"config": ("BFL_CONFIG",)},
|
||||
}
|
||||
|
||||
def generate_video(
|
||||
self,
|
||||
prompt,
|
||||
start_video,
|
||||
resolution,
|
||||
duration,
|
||||
aspect_ratio,
|
||||
generate_audio,
|
||||
safety_tolerance,
|
||||
draft,
|
||||
config=None,
|
||||
):
|
||||
arguments = self.build_arguments(
|
||||
"v2v", prompt, resolution, duration, aspect_ratio, generate_audio, safety_tolerance, draft
|
||||
)
|
||||
arguments["start_video"] = start_video
|
||||
return self.generate_video_request(arguments, config)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = (
|
||||
{
|
||||
"Flux3VideoT2V_BFL": Flux3VideoT2V,
|
||||
"Flux3VideoI2V_BFL": Flux3VideoI2V,
|
||||
"Flux3VideoV2V_BFL": Flux3VideoV2V,
|
||||
"Flux3Keyframes_BFL": Flux3Keyframes,
|
||||
}
|
||||
if VideoFromFile
|
||||
else {}
|
||||
)
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = (
|
||||
{
|
||||
"Flux3VideoT2V_BFL": "Flux 3 Video T2V (BFL)",
|
||||
"Flux3VideoI2V_BFL": "Flux 3 Video I2V (BFL)",
|
||||
"Flux3VideoV2V_BFL": "Flux 3 Video V2V (BFL)",
|
||||
"Flux3Keyframes_BFL": "Flux 3 Keyframes (BFL)",
|
||||
}
|
||||
if VideoFromFile
|
||||
else {}
|
||||
)
|
||||
+46
-2
@@ -1,5 +1,7 @@
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
@@ -35,6 +37,48 @@ class ImageToBase64:
|
||||
return (b64,)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"ImageToBase64_BFL": ImageToBase64}
|
||||
class VideoToBase64:
|
||||
"""Convert a ComfyUI VIDEO (LoadVideo output or a Flux 3 Video result) to a base64 MP4 string.
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"ImageToBase64_BFL": "Image to Base64 (BFL)"}
|
||||
Reads the video's stream source directly when it is already an MP4 in
|
||||
memory or on disk; anything else is remuxed to MP4 via VideoInput.save_to.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"video": ("VIDEO",),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("STRING",)
|
||||
FUNCTION = "convert"
|
||||
CATEGORY = "BFL/Utils"
|
||||
|
||||
def convert(self, video):
|
||||
source = video.get_stream_source() if hasattr(video, "get_stream_source") else None
|
||||
if isinstance(source, io.BytesIO):
|
||||
data = source.getvalue()
|
||||
elif isinstance(source, str) and source.lower().endswith(".mp4"):
|
||||
with open(source, "rb") as f:
|
||||
data = f.read()
|
||||
else:
|
||||
fd, temp_path = tempfile.mkstemp(suffix=".mp4")
|
||||
os.close(fd)
|
||||
try:
|
||||
video.save_to(temp_path)
|
||||
with open(temp_path, "rb") as f:
|
||||
data = f.read()
|
||||
finally:
|
||||
os.remove(temp_path)
|
||||
print(f"[BFL] Video encoded to base64 ({len(data) / (1024 * 1024):.1f} MB)")
|
||||
return (base64.b64encode(data).decode("utf-8"),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"ImageToBase64_BFL": ImageToBase64, "VideoToBase64_BFL": VideoToBase64}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"ImageToBase64_BFL": "Image to Base64 (BFL)",
|
||||
"VideoToBase64_BFL": "Video to Base64 (BFL)",
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user