diff --git a/README.md b/README.md index 0e51dc8..08cce61 100644 --- a/README.md +++ b/README.md @@ -45,29 +45,35 @@ Most of the nodes allow you to save locally the output video. If you keep the de For images, the node will always save the image locally, but you can set the `filename` to save it with a custom name. +**Notes: Duration and resolution parameters are only supported for Ray 2 and Ray 2 Flash models.** + ### LumaAIClient This node is used to create a LumaAI client. -### Ray2Text2Video - -This node is used to generate a video from a text prompt with the latest and most powerful model, Ray2. - ### LumaText2Video -This node is used to generate a video from a text prompt. +This node is used to generate a video from a text prompt. You can now choose between different models and resolutions. ### LumaImage2Video -This node is used to generate a video from an image. The image can be used as the first or last frame. +This node is used to generate a video from an image. The image can be used as the first or last frame. You can now choose between different models and resolutions. ### LumaInterpolateGenerations -This node is used to interpolate between two generations. +This node is used to interpolate between two generations. You can now choose between different models and resolutions. ### LumaExtendGeneration -This node is used to extend a generation. You can choose to extend before or after the generation. +This node is used to extend a generation. You can choose to extend before or after the generation. You can now choose between different models and resolutions. + +### LumaUpscaleGeneration + +This node is used to upscale a generation. + +### LumaAddAudio2Video + +This node is used to add audio to a video. When using this node with `Preview Video` node, you will notice that the audio won't play in comfy. For that, right click on the video and select `Open Image`, which will open the video in a new tab and you will be able to hear the audio. ### LumaPreviewVideo diff --git a/py/luma_nodes.py b/py/luma_nodes.py index f41633f..98bbef6 100644 --- a/py/luma_nodes.py +++ b/py/luma_nodes.py @@ -10,7 +10,8 @@ from .lumaai_api_node import ( CharacterReference, ImageGeneration, ModifyImage, - Ray2Text2Video, + AddAudio2Video, + UpscaleGeneration, ) from .imgbb_node import ImgBBUpload @@ -27,7 +28,8 @@ NODE_CLASS_MAPPINGS = { "CharacterReference": CharacterReference, "LumaImageGeneration": ImageGeneration, "LumaModifyImage": ModifyImage, - "Ray2Text2Video": Ray2Text2Video, + "LumaAddAudio2Video": AddAudio2Video, + "LumaUpscaleGeneration": UpscaleGeneration, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -43,5 +45,6 @@ NODE_DISPLAY_NAME_MAPPINGS = { "LumaCharacterReference": "Character Reference", "LumaImageGeneration": "Image Generation", "LumaModifyImage": "Modify Image", - "Ray2Text2Video": "Ray2 Text to Video", + "LumaAddAudio2Video": "Add Audio to Video", + "LumaUpscaleGeneration": "Upscale Generation", } diff --git a/py/lumaai_api_node.py b/py/lumaai_api_node.py index 140734f..98d932b 100644 --- a/py/lumaai_api_node.py +++ b/py/lumaai_api_node.py @@ -53,6 +53,27 @@ def parse_filename(filename): return directory, filename +def wait_for_generation(client, generation_id, save, filename, output_dir): + completed = False + while not completed: + generation = client.generations.get(id=generation_id) + if generation.state == "completed": + completed = True + elif generation.state == "failed": + raise ValueError(f"Generation failed: {generation.failure_reason}") + time.sleep(3) + + video_url = generation.assets.video + if save: + directory, filename = parse_filename(filename) + if filename == "": + filename = generation_id + download_file( + video_url, os.path.join(output_dir, directory, filename + ".mp4") + ) + return video_url + + class LumaAIClient: @classmethod def INPUT_TYPES(cls): @@ -87,78 +108,6 @@ class LumaAIClient: return (client,) - -class Ray2Text2Video: - def __init__(self): - self.output_dir = folder_paths.get_output_directory() - - @classmethod - def IS_CHANGED(cls, *args, **kwargs): - return float("NaN") - - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "client": ("LUMACLIENT", {"forceInput": True}), - "prompt": ("STRING", {"multiline": True, "default": ""}), - "duration": (["5s", "9s"], ), - "resolution": (["540p", "720p"], ), - "loop": ("BOOLEAN", {"default": False}), - "aspect_ratio": (["9:16", "3:4", "1:1", "4:3", "16:9", "21:9"],), - "save": ("BOOLEAN", {"default": True}), - }, - "optional": {"filename": ("STRING", {"default": ""})}, - } - - RETURN_TYPES = ("STRING", "STRING") - RETURN_NAMES = ("video_url", "generation_id") - OUTPUT_NODE = True - FUNCTION = "run" - CATEGORY = "LumaAI/Ray2" - - def run(self, client, prompt, duration, resolution, loop, aspect_ratio, save, filename): - """ - Generate a video from a text prompt. - """ - if prompt == "": - raise ValueError("Prompt is required") - - generation = client.generations.create( - prompt=prompt, - model="ray-2", - loop=loop, - aspect_ratio=aspect_ratio, - duration=duration, - resolution=resolution, - ) - generation_id = generation.id - completed = False - while not completed: - generation = client.generations.get(id=generation_id) - if generation.state == "completed": - completed = True - elif generation.state == "failed": - raise ValueError(f"Generation failed: {generation.failure_reason}") - time.sleep(4) - - video_url = generation.assets.video - if save: - directory, filename = parse_filename(filename) - if filename == "": - filename = generation_id - download_file( - video_url, os.path.join(self.output_dir, directory, filename + ".mp4") - ) - - return { - "ui": {"text": [generation_id]}, - "result": ( - video_url, - generation_id, - ), - } - class Text2Video: def __init__(self): self.output_dir = folder_paths.get_output_directory() @@ -172,9 +121,12 @@ class Text2Video: return { "required": { "client": ("LUMACLIENT", {"forceInput": True}), + "model": (["ray-flash-2", "ray-2", "ray-1.6"],), "prompt": ("STRING", {"multiline": True, "default": ""}), + "duration": (["5s", "9s"],), "loop": ("BOOLEAN", {"default": False}), "aspect_ratio": (["9:16", "3:4", "1:1", "4:3", "16:9", "21:9"],), + "resolution": (["540p", "720p"],), "save": ("BOOLEAN", {"default": True}), }, "optional": {"filename": ("STRING", {"default": ""})}, @@ -184,9 +136,9 @@ class Text2Video: RETURN_NAMES = ("video_url", "generation_id") OUTPUT_NODE = True FUNCTION = "run" - CATEGORY = "LumaAI/Ray1.6" + CATEGORY = "LumaAI/Ray" - def run(self, client, prompt, loop, aspect_ratio, save, filename): + def run(self, client, model, prompt, duration, loop, aspect_ratio, resolution, save, filename): """ Generate a video from a text prompt. """ @@ -194,26 +146,15 @@ class Text2Video: raise ValueError("Prompt is required") generation = client.generations.create( - prompt=prompt, loop=loop, aspect_ratio=aspect_ratio + prompt=prompt, + model=model, + loop=loop, + aspect_ratio=aspect_ratio, + duration=duration, + resolution=resolution, ) generation_id = generation.id - completed = False - while not completed: - generation = client.generations.get(id=generation_id) - if generation.state == "completed": - completed = True - elif generation.state == "failed": - raise ValueError(f"Generation failed: {generation.failure_reason}") - time.sleep(3) - - video_url = generation.assets.video - if save: - directory, filename = parse_filename(filename) - if filename == "": - filename = generation_id - download_file( - video_url, os.path.join(self.output_dir, directory, filename + ".mp4") - ) + video_url = wait_for_generation(client, generation_id, save, filename, self.output_dir) return { "ui": {"text": [generation_id]}, @@ -238,7 +179,10 @@ class Image2Video: "required": { "client": ("LUMACLIENT", {"forceInput": True}), "prompt": ("STRING", {"multiline": True, "default": ""}), + "model": (["ray-flash-2", "ray-2", "ray-1.6"],), + "duration": (["5s", "9s"],), "loop": ("BOOLEAN", {"default": False}), + "resolution": (["540p", "720p"],), "save": ("BOOLEAN", {"default": True}), }, "optional": { @@ -251,13 +195,16 @@ class Image2Video: RETURN_TYPES = ("STRING", "STRING") RETURN_NAMES = ("video_url", "generation_id") FUNCTION = "run" - CATEGORY = "LumaAI/Ray1.6" + CATEGORY = "LumaAI/Ray" def run( self, client, + model, prompt, + duration, loop, + resolution, save, init_image_url="", final_image_url="", @@ -276,26 +223,15 @@ class Image2Video: keyframes["frame1"] = {"type": "image", "url": final_image_url} generation = client.generations.create( - prompt=prompt, loop=loop, keyframes=keyframes + prompt=prompt, + model=model, + loop=loop, + duration=duration, + resolution=resolution, + keyframes=keyframes, ) generation_id = generation.id - completed = False - while not completed: - generation = client.generations.get(id=generation_id) - if generation.state == "completed": - completed = True - elif generation.state == "failed": - raise ValueError(f"Generation failed: {generation.failure_reason}") - time.sleep(3) - - video_url = generation.assets.video - if save: - directory, filename = parse_filename(filename) - if filename == "": - filename = generation_id - download_file( - video_url, os.path.join(self.output_dir, directory, filename + ".mp4") - ) + video_url = wait_for_generation(client, generation_id, save, filename, self.output_dir) return { "ui": {"text": [generation_id]}, @@ -320,6 +256,8 @@ class InterpolateGenerations: "required": { "client": ("LUMACLIENT", {"forceInput": True}), "prompt": ("STRING", {"multiline": True, "default": ""}), + "model": (["ray-flash-2", "ray-2", "ray-1.6"],), + "resolution": (["540p", "720p"],), "save": ("BOOLEAN", {"default": True}), "generation_id_1": ("STRING", {"default": "", "forceInput": True}), "generation_id_2": ("STRING", {"default": "", "forceInput": True}), @@ -330,12 +268,14 @@ class InterpolateGenerations: RETURN_TYPES = ("STRING", "STRING") RETURN_NAMES = ("video_url", "generation_id") FUNCTION = "run" - CATEGORY = "LumaAI/Ray1.6" + CATEGORY = "LumaAI/Ray" def run( self, client, + model, prompt, + resolution, save, generation_id_1, generation_id_2, @@ -353,25 +293,11 @@ class InterpolateGenerations: "frame0": {"type": "generation", "id": generation_id_1}, "frame1": {"type": "generation", "id": generation_id_2}, }, + model=model, + resolution=resolution, ) generation_id = generation.id - completed = False - while not completed: - generation = client.generations.get(id=generation_id) - if generation.state == "completed": - completed = True - elif generation.state == "failed": - raise ValueError(f"Generation failed: {generation.failure_reason}") - time.sleep(3) - - video_url = generation.assets.video - if save: - directory, filename = parse_filename(filename) - if filename == "": - filename = generation_id - download_file( - video_url, os.path.join(self.output_dir, directory, filename + ".mp4") - ) + video_url = wait_for_generation(client, generation_id, save, filename, self.output_dir) return { "ui": {"text": [generation_id]}, @@ -396,6 +322,9 @@ class ExtendGeneration: "required": { "client": ("LUMACLIENT", {"forceInput": True}), "prompt": ("STRING", {"multiline": True, "default": ""}), + "model": (["ray-flash-2", "ray-2", "ray-1.6"],), + "loop": ("BOOLEAN", {"default": False}), + "resolution": (["540p", "720p"],), "save": ("BOOLEAN", {"default": True}), }, "optional": { @@ -410,12 +339,15 @@ class ExtendGeneration: RETURN_TYPES = ("STRING", "STRING") RETURN_NAMES = ("video_url", "generation_id") FUNCTION = "run" - CATEGORY = "LumaAI/Ray1.6" + CATEGORY = "LumaAI/Ray" def run( self, client, + model, prompt, + loop, + resolution, save, init_image_url="", final_image_url="", @@ -447,31 +379,121 @@ class ExtendGeneration: if final_generation_id != "": keyframes["frame1"] = {"type": "generation", "id": final_generation_id} - generation = client.generations.create(prompt=prompt, keyframes=keyframes) - new_generation_id = generation.id - completed = False - while not completed: - generation = client.generations.get(id=new_generation_id) - if generation.state == "completed": - completed = True - elif generation.state == "failed": - raise ValueError(f"Generation failed: {generation.failure_reason}") - time.sleep(3) - - video_url = generation.assets.video - if save: - directory, filename = parse_filename(filename) - if filename == "": - filename = new_generation_id - download_file( - video_url, os.path.join(self.output_dir, directory, filename + ".mp4") - ) + generation = client.generations.create( + prompt=prompt, + model=model, + loop=loop, + resolution=resolution, + keyframes=keyframes, + ) + generation_id = generation.id + video_url = wait_for_generation(client, generation_id, save, filename, self.output_dir) return { - "ui": {"text": [new_generation_id]}, + "ui": {"text": [generation_id]}, "result": ( video_url, - new_generation_id, + generation_id, + ), + } + + +class UpscaleGeneration: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("LUMACLIENT", {"forceInput": True}), + "generation_id": ("STRING", {"default": "", "forceInput": True}), + "resolution": (["540p", "720p", "1080p", "4k"],), + "save": ("BOOLEAN", {"default": True}), + }, + "optional": { + "filename": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("video_url", "generation_id") + FUNCTION = "run" + CATEGORY = "LumaAI/Upscale" + + def run( + self, + client, + generation_id, + resolution, + save, + filename="", + ): + """ + Upscale a generation. + """ + generation = client.generations.upscale(id=generation_id, resolution=resolution) + upscaled_generation_id = generation.id + video_url = wait_for_generation(client, upscaled_generation_id, save, filename, self.output_dir) + + return { + "ui": {"text": [upscaled_generation_id]}, + "result": ( + video_url, + upscaled_generation_id, + ), + } + + +class AddAudio2Video: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + + @classmethod + def IS_CHANGED(cls, *args, **kwargs): + return float("NaN") + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("LUMACLIENT", {"forceInput": True}), + "generation_id": ("STRING", {"default": "", "forceInput": True}), + "prompt": ("STRING", {"multiline": True, "default": ""}), + "negative_prompt": ("STRING", {"multiline": True, "default": ""}), + "save": ("BOOLEAN", {"default": True}), + }, + "optional": { + "filename": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("video_url", "generation_id") + FUNCTION = "run" + CATEGORY = "LumaAI/Audio" + + def run( + self, + client, + generation_id, + prompt, + negative_prompt, + save, + filename="", + ): + """ + Upscale a generation. + """ + generation = client.generations.audio(id=generation_id, prompt=prompt, negative_prompt=negative_prompt) + with_audio_generation_id = generation.id + video_url = wait_for_generation(client, with_audio_generation_id, save, filename, self.output_dir) + + return { + "ui": {"text": [with_audio_generation_id]}, + "result": ( + video_url, + with_audio_generation_id, ), } @@ -485,6 +507,10 @@ class PreviewVideo: } } + @classmethod + def IS_CHANGED(cls, *args, **kwargs): + return float("NaN") + OUTPUT_NODE = True FUNCTION = "run" CATEGORY = "LumaAI/Utils" diff --git a/pyproject.toml b/pyproject.toml index 152b68c..f72b557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "comfyui-lumaai-api" description = "This custom node allows you to use the LumaAI API." -version = "1.0.6" +version = "1.1.0" license = {file = "LICENSE"} -dependencies = ["lumaai==1.3.0", "requests", "numpy", "Pillow"] +dependencies = ["lumaai==1.7.0", "requests", "numpy", "Pillow"] [project.urls] Repository = "https://github.com/lumalabs/ComfyUI-LumaAI-API" diff --git a/requirements.txt b/requirements.txt index 2ea9aa6..f59240e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -lumaai==1.3.0 +lumaai==1.7.0 requests numpy Pillow \ No newline at end of file diff --git a/web/js/showGenerationId.js b/web/js/showGenerationId.js index cd630de..0abe386 100644 --- a/web/js/showGenerationId.js +++ b/web/js/showGenerationId.js @@ -37,7 +37,7 @@ app.registerExtension({ sz[1] = this.size[1]; } this.onResize?.(sz); - app.graph.setDirtyCanvas(true, false); + app.graph.setDirtyCanvas(true, true); }); } diff --git a/workflows/AddAudio2Video.json b/workflows/AddAudio2Video.json new file mode 100644 index 0000000..55728eb --- /dev/null +++ b/workflows/AddAudio2Video.json @@ -0,0 +1,227 @@ +{ + "last_node_id": 5, + "last_link_id": 5, + "nodes": [ + { + "id": 2, + "type": "LumaText2Video", + "pos": [ + 460.42974853515625, + 274.5 + ], + "size": [ + 400, + 364 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "link": 1 + } + ], + "outputs": [ + { + "name": "video_url", + "type": "STRING", + "links": [] + }, + { + "name": "generation_id", + "type": "STRING", + "links": [ + 3 + ], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LumaText2Video" + }, + "widgets_values": [ + "ray-flash-2", + "a car", + "5s", + false, + "9:16", + "540p", + true, + "", + "ee4e9dad-9f1e-4e5e-bc37-1b46e9825750", + "ee4e9dad-9f1e-4e5e-bc37-1b46e9825750" + ] + }, + { + "id": 1, + "type": "LumaAddAudio2Video", + "pos": [ + 1003.0374755859375, + 442.3060607910156 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "link": 5 + }, + { + "name": "generation_id", + "type": "STRING", + "link": 3, + "widget": { + "name": "generation_id" + } + } + ], + "outputs": [ + { + "name": "video_url", + "type": "STRING", + "links": [ + 4 + ] + }, + { + "name": "generation_id", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "LumaAddAudio2Video" + }, + "widgets_values": [ + "", + "noise from a car", + "", + true, + "" + ] + }, + { + "id": 5, + "type": "LumaPreviewVideo", + "pos": [ + 1583.6092529296875, + 145.96243286132812 + ], + "size": [ + 768, + 768 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "video_url", + "type": "STRING", + "link": 4, + "widget": { + "name": "video_url" + } + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "LumaPreviewVideo" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 3, + "type": "LumaAIClient", + "pos": [ + 32.32486343383789, + 344.1593933105469 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "links": [ + 1, + 5 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LumaAIClient" + }, + "widgets_values": [ + "" + ] + } + ], + "links": [ + [ + 1, + 3, + 0, + 2, + 0, + "LUMACLIENT" + ], + [ + 3, + 2, + 1, + 1, + 1, + "STRING" + ], + [ + 4, + 1, + 0, + 5, + 0, + "STRING" + ], + [ + 5, + 3, + 0, + 1, + 0, + "LUMACLIENT" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.5559917313492242, + "offset": [ + 230.56296338473092, + 95.73040355967112 + ] + }, + "node_versions": { + "ComfyUI-LumaAI-API": "a53576c98bcffffca5474451a8ca5de4faf92943" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/workflows/Upscale.json b/workflows/Upscale.json new file mode 100644 index 0000000..c66ecc9 --- /dev/null +++ b/workflows/Upscale.json @@ -0,0 +1,227 @@ +{ + "last_node_id": 6, + "last_link_id": 9, + "nodes": [ + { + "id": 5, + "type": "LumaPreviewVideo", + "pos": [ + 1583.6092529296875, + 145.96243286132812 + ], + "size": [ + 768, + 768 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "video_url", + "type": "STRING", + "link": 8, + "widget": { + "name": "video_url" + } + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "LumaPreviewVideo" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 2, + "type": "LumaText2Video", + "pos": [ + 460.42974853515625, + 274.5 + ], + "size": [ + 400, + 364 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "link": 1 + } + ], + "outputs": [ + { + "name": "video_url", + "type": "STRING", + "links": [] + }, + { + "name": "generation_id", + "type": "STRING", + "links": [ + 7 + ], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LumaText2Video" + }, + "widgets_values": [ + "ray-flash-2", + "a car", + "5s", + false, + "9:16", + "540p", + true, + "", + "24ed0260-bc59-419f-8f10-794d291176d0", + "24ed0260-bc59-419f-8f10-794d291176d0" + ] + }, + { + "id": 6, + "type": "LumaUpscaleGeneration", + "pos": [ + 1056.1300048828125, + 351.3099365234375 + ], + "size": [ + 315, + 150 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "link": 9 + }, + { + "name": "generation_id", + "type": "STRING", + "link": 7, + "widget": { + "name": "generation_id" + } + } + ], + "outputs": [ + { + "name": "video_url", + "type": "STRING", + "links": [ + 8 + ], + "slot_index": 0 + }, + { + "name": "generation_id", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "LumaUpscaleGeneration" + }, + "widgets_values": [ + "", + "1080p", + true, + "" + ] + }, + { + "id": 3, + "type": "LumaAIClient", + "pos": [ + 32.32486343383789, + 344.1593933105469 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "links": [ + 1, + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LumaAIClient" + }, + "widgets_values": [ + "" + ] + } + ], + "links": [ + [ + 1, + 3, + 0, + 2, + 0, + "LUMACLIENT" + ], + [ + 7, + 2, + 1, + 6, + 1, + "STRING" + ], + [ + 8, + 6, + 0, + 5, + 0, + "STRING" + ], + [ + 9, + 3, + 0, + 6, + 0, + "LUMACLIENT" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.9849732675807652, + "offset": [ + -92.60513941852872, + -55.773528450482495 + ] + }, + "node_versions": { + "ComfyUI-LumaAI-API": "a53576c98bcffffca5474451a8ca5de4faf92943" + } + }, + "version": 0.4 +} \ No newline at end of file