5 Commits
Author SHA1 Message Date
Pedro Gabriel Gengo Lourenço d783188eeb Merge pull request #12 from lumalabs/feature/adding-im2vid-and-flash-and-audio
Adding support for ray 2 and ray2 flash in all tasks, and adding audi…
2025-03-24 22:17:42 -03:00
Pedro gengo 822248eb37 Updating version 2025-03-24 22:11:12 -03:00
Pedro gengo 07dd99197e Small fixes + updating documentation 2025-03-24 22:09:47 -03:00
Pedro gengo a53576c98b Exposing new nodes 2025-03-24 21:33:24 -03:00
Pedro gengo 04c8f37f73 Adding support for ray 2 and ray2 flash in all tasks, and adding audio and upscale tasks 2025-03-24 21:25:38 -03:00
8 changed files with 655 additions and 166 deletions
+14 -8
View File
@@ -45,29 +45,35 @@ Most of the nodes allow you to save locally the output video. If you keep the de
For images, the node will always save the image locally, but you can set the `filename` to save it with a custom name.
**Notes: Duration and resolution parameters are only supported for Ray 2 and Ray 2 Flash models.**
### LumaAIClient
This node is used to create a LumaAI client.
### Ray2Text2Video
This node is used to generate a video from a text prompt with the latest and most powerful model, Ray2.
### LumaText2Video
This node is used to generate a video from a text prompt.
This node is used to generate a video from a text prompt. You can now choose between different models and resolutions.
### LumaImage2Video
This node is used to generate a video from an image. The image can be used as the first or last frame.
This node is used to generate a video from an image. The image can be used as the first or last frame. You can now choose between different models and resolutions.
### LumaInterpolateGenerations
This node is used to interpolate between two generations.
This node is used to interpolate between two generations. You can now choose between different models and resolutions.
### LumaExtendGeneration
This node is used to extend a generation. You can choose to extend before or after the generation.
This node is used to extend a generation. You can choose to extend before or after the generation. You can now choose between different models and resolutions.
### LumaUpscaleGeneration
This node is used to upscale a generation.
### LumaAddAudio2Video
This node is used to add audio to a video. When using this node with `Preview Video` node, you will notice that the audio won't play in comfy. For that, right click on the video and select `Open Image`, which will open the video in a new tab and you will be able to hear the audio.
### LumaPreviewVideo
+6 -3
View File
@@ -10,7 +10,8 @@ from .lumaai_api_node import (
CharacterReference,
ImageGeneration,
ModifyImage,
Ray2Text2Video,
AddAudio2Video,
UpscaleGeneration,
)
from .imgbb_node import ImgBBUpload
@@ -27,7 +28,8 @@ NODE_CLASS_MAPPINGS = {
"CharacterReference": CharacterReference,
"LumaImageGeneration": ImageGeneration,
"LumaModifyImage": ModifyImage,
"Ray2Text2Video": Ray2Text2Video,
"LumaAddAudio2Video": AddAudio2Video,
"LumaUpscaleGeneration": UpscaleGeneration,
}
NODE_DISPLAY_NAME_MAPPINGS = {
@@ -43,5 +45,6 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"LumaCharacterReference": "Character Reference",
"LumaImageGeneration": "Image Generation",
"LumaModifyImage": "Modify Image",
"Ray2Text2Video": "Ray2 Text to Video",
"LumaAddAudio2Video": "Add Audio to Video",
"LumaUpscaleGeneration": "Upscale Generation",
}
+177 -151
View File
@@ -53,6 +53,27 @@ def parse_filename(filename):
return directory, filename
def wait_for_generation(client, generation_id, save, filename, output_dir):
completed = False
while not completed:
generation = client.generations.get(id=generation_id)
if generation.state == "completed":
completed = True
elif generation.state == "failed":
raise ValueError(f"Generation failed: {generation.failure_reason}")
time.sleep(3)
video_url = generation.assets.video
if save:
directory, filename = parse_filename(filename)
if filename == "":
filename = generation_id
download_file(
video_url, os.path.join(output_dir, directory, filename + ".mp4")
)
return video_url
class LumaAIClient:
@classmethod
def INPUT_TYPES(cls):
@@ -87,78 +108,6 @@ class LumaAIClient:
return (client,)
class Ray2Text2Video:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def IS_CHANGED(cls, *args, **kwargs):
return float("NaN")
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"client": ("LUMACLIENT", {"forceInput": True}),
"prompt": ("STRING", {"multiline": True, "default": ""}),
"duration": (["5s", "9s"], ),
"resolution": (["540p", "720p"], ),
"loop": ("BOOLEAN", {"default": False}),
"aspect_ratio": (["9:16", "3:4", "1:1", "4:3", "16:9", "21:9"],),
"save": ("BOOLEAN", {"default": True}),
},
"optional": {"filename": ("STRING", {"default": ""})},
}
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("video_url", "generation_id")
OUTPUT_NODE = True
FUNCTION = "run"
CATEGORY = "LumaAI/Ray2"
def run(self, client, prompt, duration, resolution, loop, aspect_ratio, save, filename):
"""
Generate a video from a text prompt.
"""
if prompt == "":
raise ValueError("Prompt is required")
generation = client.generations.create(
prompt=prompt,
model="ray-2",
loop=loop,
aspect_ratio=aspect_ratio,
duration=duration,
resolution=resolution,
)
generation_id = generation.id
completed = False
while not completed:
generation = client.generations.get(id=generation_id)
if generation.state == "completed":
completed = True
elif generation.state == "failed":
raise ValueError(f"Generation failed: {generation.failure_reason}")
time.sleep(4)
video_url = generation.assets.video
if save:
directory, filename = parse_filename(filename)
if filename == "":
filename = generation_id
download_file(
video_url, os.path.join(self.output_dir, directory, filename + ".mp4")
)
return {
"ui": {"text": [generation_id]},
"result": (
video_url,
generation_id,
),
}
class Text2Video:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@@ -172,9 +121,12 @@ class Text2Video:
return {
"required": {
"client": ("LUMACLIENT", {"forceInput": True}),
"model": (["ray-flash-2", "ray-2", "ray-1.6"],),
"prompt": ("STRING", {"multiline": True, "default": ""}),
"duration": (["5s", "9s"],),
"loop": ("BOOLEAN", {"default": False}),
"aspect_ratio": (["9:16", "3:4", "1:1", "4:3", "16:9", "21:9"],),
"resolution": (["540p", "720p"],),
"save": ("BOOLEAN", {"default": True}),
},
"optional": {"filename": ("STRING", {"default": ""})},
@@ -184,9 +136,9 @@ class Text2Video:
RETURN_NAMES = ("video_url", "generation_id")
OUTPUT_NODE = True
FUNCTION = "run"
CATEGORY = "LumaAI/Ray1.6"
CATEGORY = "LumaAI/Ray"
def run(self, client, prompt, loop, aspect_ratio, save, filename):
def run(self, client, model, prompt, duration, loop, aspect_ratio, resolution, save, filename):
"""
Generate a video from a text prompt.
"""
@@ -194,26 +146,15 @@ class Text2Video:
raise ValueError("Prompt is required")
generation = client.generations.create(
prompt=prompt, loop=loop, aspect_ratio=aspect_ratio
prompt=prompt,
model=model,
loop=loop,
aspect_ratio=aspect_ratio,
duration=duration,
resolution=resolution,
)
generation_id = generation.id
completed = False
while not completed:
generation = client.generations.get(id=generation_id)
if generation.state == "completed":
completed = True
elif generation.state == "failed":
raise ValueError(f"Generation failed: {generation.failure_reason}")
time.sleep(3)
video_url = generation.assets.video
if save:
directory, filename = parse_filename(filename)
if filename == "":
filename = generation_id
download_file(
video_url, os.path.join(self.output_dir, directory, filename + ".mp4")
)
video_url = wait_for_generation(client, generation_id, save, filename, self.output_dir)
return {
"ui": {"text": [generation_id]},
@@ -238,7 +179,10 @@ class Image2Video:
"required": {
"client": ("LUMACLIENT", {"forceInput": True}),
"prompt": ("STRING", {"multiline": True, "default": ""}),
"model": (["ray-flash-2", "ray-2", "ray-1.6"],),
"duration": (["5s", "9s"],),
"loop": ("BOOLEAN", {"default": False}),
"resolution": (["540p", "720p"],),
"save": ("BOOLEAN", {"default": True}),
},
"optional": {
@@ -251,13 +195,16 @@ class Image2Video:
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("video_url", "generation_id")
FUNCTION = "run"
CATEGORY = "LumaAI/Ray1.6"
CATEGORY = "LumaAI/Ray"
def run(
self,
client,
model,
prompt,
duration,
loop,
resolution,
save,
init_image_url="",
final_image_url="",
@@ -276,26 +223,15 @@ class Image2Video:
keyframes["frame1"] = {"type": "image", "url": final_image_url}
generation = client.generations.create(
prompt=prompt, loop=loop, keyframes=keyframes
prompt=prompt,
model=model,
loop=loop,
duration=duration,
resolution=resolution,
keyframes=keyframes,
)
generation_id = generation.id
completed = False
while not completed:
generation = client.generations.get(id=generation_id)
if generation.state == "completed":
completed = True
elif generation.state == "failed":
raise ValueError(f"Generation failed: {generation.failure_reason}")
time.sleep(3)
video_url = generation.assets.video
if save:
directory, filename = parse_filename(filename)
if filename == "":
filename = generation_id
download_file(
video_url, os.path.join(self.output_dir, directory, filename + ".mp4")
)
video_url = wait_for_generation(client, generation_id, save, filename, self.output_dir)
return {
"ui": {"text": [generation_id]},
@@ -320,6 +256,8 @@ class InterpolateGenerations:
"required": {
"client": ("LUMACLIENT", {"forceInput": True}),
"prompt": ("STRING", {"multiline": True, "default": ""}),
"model": (["ray-flash-2", "ray-2", "ray-1.6"],),
"resolution": (["540p", "720p"],),
"save": ("BOOLEAN", {"default": True}),
"generation_id_1": ("STRING", {"default": "", "forceInput": True}),
"generation_id_2": ("STRING", {"default": "", "forceInput": True}),
@@ -330,12 +268,14 @@ class InterpolateGenerations:
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("video_url", "generation_id")
FUNCTION = "run"
CATEGORY = "LumaAI/Ray1.6"
CATEGORY = "LumaAI/Ray"
def run(
self,
client,
model,
prompt,
resolution,
save,
generation_id_1,
generation_id_2,
@@ -353,25 +293,11 @@ class InterpolateGenerations:
"frame0": {"type": "generation", "id": generation_id_1},
"frame1": {"type": "generation", "id": generation_id_2},
},
model=model,
resolution=resolution,
)
generation_id = generation.id
completed = False
while not completed:
generation = client.generations.get(id=generation_id)
if generation.state == "completed":
completed = True
elif generation.state == "failed":
raise ValueError(f"Generation failed: {generation.failure_reason}")
time.sleep(3)
video_url = generation.assets.video
if save:
directory, filename = parse_filename(filename)
if filename == "":
filename = generation_id
download_file(
video_url, os.path.join(self.output_dir, directory, filename + ".mp4")
)
video_url = wait_for_generation(client, generation_id, save, filename, self.output_dir)
return {
"ui": {"text": [generation_id]},
@@ -396,6 +322,9 @@ class ExtendGeneration:
"required": {
"client": ("LUMACLIENT", {"forceInput": True}),
"prompt": ("STRING", {"multiline": True, "default": ""}),
"model": (["ray-flash-2", "ray-2", "ray-1.6"],),
"loop": ("BOOLEAN", {"default": False}),
"resolution": (["540p", "720p"],),
"save": ("BOOLEAN", {"default": True}),
},
"optional": {
@@ -410,12 +339,15 @@ class ExtendGeneration:
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("video_url", "generation_id")
FUNCTION = "run"
CATEGORY = "LumaAI/Ray1.6"
CATEGORY = "LumaAI/Ray"
def run(
self,
client,
model,
prompt,
loop,
resolution,
save,
init_image_url="",
final_image_url="",
@@ -447,31 +379,121 @@ class ExtendGeneration:
if final_generation_id != "":
keyframes["frame1"] = {"type": "generation", "id": final_generation_id}
generation = client.generations.create(prompt=prompt, keyframes=keyframes)
new_generation_id = generation.id
completed = False
while not completed:
generation = client.generations.get(id=new_generation_id)
if generation.state == "completed":
completed = True
elif generation.state == "failed":
raise ValueError(f"Generation failed: {generation.failure_reason}")
time.sleep(3)
video_url = generation.assets.video
if save:
directory, filename = parse_filename(filename)
if filename == "":
filename = new_generation_id
download_file(
video_url, os.path.join(self.output_dir, directory, filename + ".mp4")
)
generation = client.generations.create(
prompt=prompt,
model=model,
loop=loop,
resolution=resolution,
keyframes=keyframes,
)
generation_id = generation.id
video_url = wait_for_generation(client, generation_id, save, filename, self.output_dir)
return {
"ui": {"text": [new_generation_id]},
"ui": {"text": [generation_id]},
"result": (
video_url,
new_generation_id,
generation_id,
),
}
class UpscaleGeneration:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"client": ("LUMACLIENT", {"forceInput": True}),
"generation_id": ("STRING", {"default": "", "forceInput": True}),
"resolution": (["540p", "720p", "1080p", "4k"],),
"save": ("BOOLEAN", {"default": True}),
},
"optional": {
"filename": ("STRING", {"default": ""}),
},
}
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("video_url", "generation_id")
FUNCTION = "run"
CATEGORY = "LumaAI/Upscale"
def run(
self,
client,
generation_id,
resolution,
save,
filename="",
):
"""
Upscale a generation.
"""
generation = client.generations.upscale(id=generation_id, resolution=resolution)
upscaled_generation_id = generation.id
video_url = wait_for_generation(client, upscaled_generation_id, save, filename, self.output_dir)
return {
"ui": {"text": [upscaled_generation_id]},
"result": (
video_url,
upscaled_generation_id,
),
}
class AddAudio2Video:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def IS_CHANGED(cls, *args, **kwargs):
return float("NaN")
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"client": ("LUMACLIENT", {"forceInput": True}),
"generation_id": ("STRING", {"default": "", "forceInput": True}),
"prompt": ("STRING", {"multiline": True, "default": ""}),
"negative_prompt": ("STRING", {"multiline": True, "default": ""}),
"save": ("BOOLEAN", {"default": True}),
},
"optional": {
"filename": ("STRING", {"default": ""}),
},
}
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("video_url", "generation_id")
FUNCTION = "run"
CATEGORY = "LumaAI/Audio"
def run(
self,
client,
generation_id,
prompt,
negative_prompt,
save,
filename="",
):
"""
Upscale a generation.
"""
generation = client.generations.audio(id=generation_id, prompt=prompt, negative_prompt=negative_prompt)
with_audio_generation_id = generation.id
video_url = wait_for_generation(client, with_audio_generation_id, save, filename, self.output_dir)
return {
"ui": {"text": [with_audio_generation_id]},
"result": (
video_url,
with_audio_generation_id,
),
}
@@ -485,6 +507,10 @@ class PreviewVideo:
}
}
@classmethod
def IS_CHANGED(cls, *args, **kwargs):
return float("NaN")
OUTPUT_NODE = True
FUNCTION = "run"
CATEGORY = "LumaAI/Utils"
+2 -2
View File
@@ -1,9 +1,9 @@
[project]
name = "comfyui-lumaai-api"
description = "This custom node allows you to use the LumaAI API."
version = "1.0.6"
version = "1.1.0"
license = {file = "LICENSE"}
dependencies = ["lumaai==1.3.0", "requests", "numpy", "Pillow"]
dependencies = ["lumaai==1.7.0", "requests", "numpy", "Pillow"]
[project.urls]
Repository = "https://github.com/lumalabs/ComfyUI-LumaAI-API"
+1 -1
View File
@@ -1,4 +1,4 @@
lumaai==1.3.0
lumaai==1.7.0
requests
numpy
Pillow
+1 -1
View File
@@ -37,7 +37,7 @@ app.registerExtension({
sz[1] = this.size[1];
}
this.onResize?.(sz);
app.graph.setDirtyCanvas(true, false);
app.graph.setDirtyCanvas(true, true);
});
}
+227
View File
@@ -0,0 +1,227 @@
{
"last_node_id": 5,
"last_link_id": 5,
"nodes": [
{
"id": 2,
"type": "LumaText2Video",
"pos": [
460.42974853515625,
274.5
],
"size": [
400,
364
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{
"name": "client",
"type": "LUMACLIENT",
"link": 1
}
],
"outputs": [
{
"name": "video_url",
"type": "STRING",
"links": []
},
{
"name": "generation_id",
"type": "STRING",
"links": [
3
],
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "LumaText2Video"
},
"widgets_values": [
"ray-flash-2",
"a car",
"5s",
false,
"9:16",
"540p",
true,
"",
"ee4e9dad-9f1e-4e5e-bc37-1b46e9825750",
"ee4e9dad-9f1e-4e5e-bc37-1b46e9825750"
]
},
{
"id": 1,
"type": "LumaAddAudio2Video",
"pos": [
1003.0374755859375,
442.3060607910156
],
"size": [
400,
200
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "client",
"type": "LUMACLIENT",
"link": 5
},
{
"name": "generation_id",
"type": "STRING",
"link": 3,
"widget": {
"name": "generation_id"
}
}
],
"outputs": [
{
"name": "video_url",
"type": "STRING",
"links": [
4
]
},
{
"name": "generation_id",
"type": "STRING",
"links": null
}
],
"properties": {
"Node name for S&R": "LumaAddAudio2Video"
},
"widgets_values": [
"",
"noise from a car",
"",
true,
""
]
},
{
"id": 5,
"type": "LumaPreviewVideo",
"pos": [
1583.6092529296875,
145.96243286132812
],
"size": [
768,
768
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "video_url",
"type": "STRING",
"link": 4,
"widget": {
"name": "video_url"
}
}
],
"outputs": [],
"properties": {
"Node name for S&R": "LumaPreviewVideo"
},
"widgets_values": [
"",
null
]
},
{
"id": 3,
"type": "LumaAIClient",
"pos": [
32.32486343383789,
344.1593933105469
],
"size": [
315,
58
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "client",
"type": "LUMACLIENT",
"links": [
1,
5
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "LumaAIClient"
},
"widgets_values": [
""
]
}
],
"links": [
[
1,
3,
0,
2,
0,
"LUMACLIENT"
],
[
3,
2,
1,
1,
1,
"STRING"
],
[
4,
1,
0,
5,
0,
"STRING"
],
[
5,
3,
0,
1,
0,
"LUMACLIENT"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.5559917313492242,
"offset": [
230.56296338473092,
95.73040355967112
]
},
"node_versions": {
"ComfyUI-LumaAI-API": "a53576c98bcffffca5474451a8ca5de4faf92943"
}
},
"version": 0.4
}
+227
View File
@@ -0,0 +1,227 @@
{
"last_node_id": 6,
"last_link_id": 9,
"nodes": [
{
"id": 5,
"type": "LumaPreviewVideo",
"pos": [
1583.6092529296875,
145.96243286132812
],
"size": [
768,
768
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "video_url",
"type": "STRING",
"link": 8,
"widget": {
"name": "video_url"
}
}
],
"outputs": [],
"properties": {
"Node name for S&R": "LumaPreviewVideo"
},
"widgets_values": [
"",
null
]
},
{
"id": 2,
"type": "LumaText2Video",
"pos": [
460.42974853515625,
274.5
],
"size": [
400,
364
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{
"name": "client",
"type": "LUMACLIENT",
"link": 1
}
],
"outputs": [
{
"name": "video_url",
"type": "STRING",
"links": []
},
{
"name": "generation_id",
"type": "STRING",
"links": [
7
],
"slot_index": 1
}
],
"properties": {
"Node name for S&R": "LumaText2Video"
},
"widgets_values": [
"ray-flash-2",
"a car",
"5s",
false,
"9:16",
"540p",
true,
"",
"24ed0260-bc59-419f-8f10-794d291176d0",
"24ed0260-bc59-419f-8f10-794d291176d0"
]
},
{
"id": 6,
"type": "LumaUpscaleGeneration",
"pos": [
1056.1300048828125,
351.3099365234375
],
"size": [
315,
150
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "client",
"type": "LUMACLIENT",
"link": 9
},
{
"name": "generation_id",
"type": "STRING",
"link": 7,
"widget": {
"name": "generation_id"
}
}
],
"outputs": [
{
"name": "video_url",
"type": "STRING",
"links": [
8
],
"slot_index": 0
},
{
"name": "generation_id",
"type": "STRING",
"links": null
}
],
"properties": {
"Node name for S&R": "LumaUpscaleGeneration"
},
"widgets_values": [
"",
"1080p",
true,
""
]
},
{
"id": 3,
"type": "LumaAIClient",
"pos": [
32.32486343383789,
344.1593933105469
],
"size": [
315,
58
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [
{
"name": "client",
"type": "LUMACLIENT",
"links": [
1,
9
],
"slot_index": 0
}
],
"properties": {
"Node name for S&R": "LumaAIClient"
},
"widgets_values": [
""
]
}
],
"links": [
[
1,
3,
0,
2,
0,
"LUMACLIENT"
],
[
7,
2,
1,
6,
1,
"STRING"
],
[
8,
6,
0,
5,
0,
"STRING"
],
[
9,
3,
0,
6,
0,
"LUMACLIENT"
]
],
"groups": [],
"config": {},
"extra": {
"ds": {
"scale": 0.9849732675807652,
"offset": [
-92.60513941852872,
-55.773528450482495
]
},
"node_versions": {
"ComfyUI-LumaAI-API": "a53576c98bcffffca5474451a8ca5de4faf92943"
}
},
"version": 0.4
}