diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9f79387 --- /dev/null +++ b/.gitignore @@ -0,0 +1,172 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# VS Code +.vscode/ + +# macOS +.DS_Store + +# Windows +Thumbs.db + +config.ini +node.zip \ No newline at end of file diff --git a/README.md b/README.md index 41e3d48..dbfa52f 100644 --- a/README.md +++ b/README.md @@ -1 +1,82 @@ -# ComfyUI-LumaAI-API \ No newline at end of file +# ComfyUI-LumaAI-API +

+ LumaAI Logo +

+ + +This is a custom node for ComfyUI that allows you to use the Luma AI API directly in ComfyUI. Luma AI API is based on top of [Dream Machine](https://lumalabs.ai/dream-machine/api), which is a video generation model developed by Luma. For more information, see [Luma AI API Documentation](https://docs.lumalabs.ai/docs/api). + +## Requirements + +Before using this node, you need to have an LumaAI API key. Get one [here](https://lumalabs.ai/dream-machine/api). If you want to use images as input, you will need to host them. As a suggestion, you can use [ImgBB](https://api.imgbb.com/) to host your images. + +## Installation + +### Installing manually + +1. Navigate to the `ComfyUI/custom_nodes` directory. + +2. Clone this repository: + ``` + git clone https://github.com/lumalabs/ComfyUI-LumaAI-API.git + ``` + The path should be `ComfyUI/custom_nodes/ComfyUI-LumaAI-API/*`, where `*` represents all the files in this repo. + +3. Run `cd ComfyUI-LumaAI-API && pip install -r requirements.txt` to install the dependencies. + +4. If you don't want to expose your Luma API key, you can add it to the `config.ini` file and keep it empty in the node. + +5. Start ComfyUI and enjoy using the LumaAI API node! + +### Installing with ComfyUI-Manager + +1. Open ComfyUI-Manager and install the LumaAI API node (ComfyUI-LumaAI-API). + +### Installing with Comfy Registry + +1. Run `comfy node registry-install comfyui-lumaai-api` to install the node. + +## Nodes + +Most of the nodes allow you to save locally the output video. If you keep the default `filename` (empty string), the video will be saved in the `outputs` folder using the `generation_id` as the name. + +### LumaAIClient + +This node is used to create a LumaAI client. + +### LumaText2Video + +This node is used to generate a video from a text prompt. + +### LumaImage2Video + +This node is used to generate a video from an image. The image can be used as the first or last frame. + +### LumaInterpolateGenerations + +This node is used to interpolate between two generations. + +### LumaExtendGeneration + +This node is used to extend a generation. You can choose to extend before or after the generation. + +### LumaPreviewVideo + +This node is used to preview a video. The video is resized to 768px to look better on ComfyUI. + +### ImgBBUpload + +This node is used to upload an image to ImgBB and return the URL. We need this because Luma API currently only supports image urls as input. +To use this node, you need to have an ImgBB API key. Create an account and get one [here](https://api.imgbb.com/). + +## Examples + +For examples, see [workflows folder](./workflows). To use, just download the workflow json and import it into ComfyUI. + +## API Documentation + +For more information about the Luma AI API, see [Luma AI API Documentation](https://docs.lumalabs.ai/docs/api). + +## Pricing + +For pricing, see [Luma AI Pricing](https://lumalabs.ai/dream-machine/api/pricing). diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..fcf4a53 --- /dev/null +++ b/__init__.py @@ -0,0 +1,4 @@ +from .py.nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +WEB_DIRECTORY = "./web" +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/assets/luma_logo.png b/assets/luma_logo.png new file mode 100644 index 0000000..0d6e4f6 Binary files /dev/null and b/assets/luma_logo.png differ diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..ef2ed2a --- /dev/null +++ b/config.ini @@ -0,0 +1,2 @@ +[API] +LUMAAI_API_KEY = \ No newline at end of file diff --git a/py/imgbb_node.py b/py/imgbb_node.py new file mode 100644 index 0000000..74e956d --- /dev/null +++ b/py/imgbb_node.py @@ -0,0 +1,73 @@ +# Based on https://github.com/revirevy/Comfyui_saveimage_imgbb/blob/main/ImgBBUploader.py +import requests +import base64 +from PIL import Image +import io +import numpy as np + + +class ImgBBUpload: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ("IMAGE",), + "api_key": ("STRING", {"default": "", "multiline": False}), + "expire": ("BOOLEAN", {"default": False}), + "expiration_time": ( + "INT", + {"default": 60, "min": 60, "max": 15552000, "step": 1}, + ), + }, + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("image_url",) + + FUNCTION = "upload" + + CATEGORY = "image/upload" + + def upload(self, image, api_key, expire, expiration_time): + """ + Upload an image to ImgBB and return the URL. + """ + if not api_key: + raise ValueError("API Key is required") + + # Convert image tensor to PIL Image + img = image[0] + img = 255.0 * img.cpu().numpy() + img = Image.fromarray(np.clip(img, 0, 255).astype(np.uint8)) + + # Save PIL Image to buffer + buffer = io.BytesIO() + img.save(buffer, format="PNG") + buffer.seek(0) + + base64_image = base64.b64encode(buffer.getvalue()).decode("utf-8") + + url = f"https://api.imgbb.com/1/upload?key={api_key}" + if expire: + url += f"&expiration={expiration_time}" + + payload = { + "image": base64_image, + } + + try: + response = requests.post(url, data=payload) + result = response.json() + + if result.get("success"): + return result["data"]["url"], result["data"]["delete_url"] + else: + error_message = result.get("error", {}).get("message", "Unknown error") + raise ValueError(f"Error: {error_message}") + except Exception as e: + raise ValueError(f"Error: {str(e)}") + + +NODE_CLASS_MAPPINGS = {"ImgBBUpload": ImgBBUpload} + +NODE_DISPLAY_NAME_MAPPINGS = {"ImgBBUpload": "Upload to ImgBB"} diff --git a/py/lumaai_api_node.py b/py/lumaai_api_node.py new file mode 100644 index 0000000..c533f44 --- /dev/null +++ b/py/lumaai_api_node.py @@ -0,0 +1,401 @@ +import configparser +import os +import requests +import time +from lumaai import LumaAI + +import folder_paths + +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +config_path = os.path.join(parent_dir, "config.ini") + +config = configparser.ConfigParser() +config.read(config_path) + +try: + luma_api_key = config['API']['LUMAAI_API_KEY'] + if luma_api_key != "": + os.environ["LUMAAI_API_KEY"] = luma_api_key + else: + print("Warning: LUMAAI_API_KEY is empty in config.ini") +except KeyError: + print("Error: LUMAAI_API_KEY not found in config.ini") + + +def download_file(url, file_name): + response = requests.get(url, stream=True) + with open(file_name, "wb") as file: + file.write(response.content) + print(f"File downloaded as {file_name}") + +def parse_filename(filename): + # Remove file extension if present + filename = os.path.splitext(filename)[0] + + # Check if the filename ends with a directory separator + if filename.endswith(os.path.sep): + # If it does, we'll use the generation_id as the filename later + directory = filename + filename = "" + else: + # Otherwise, split the path into directory and filename + directory, filename = os.path.split(filename) + + # Create directories if they don't exist + output_path = folder_paths.get_output_directory() + full_directory = os.path.join(output_path, directory) + if directory and not os.path.exists(full_directory): + os.makedirs(full_directory) + + return directory, filename + + +class LumaAIClient: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "api_key": ( + "STRING", + { + "default": "", + }, + ) + }, + } + + RETURN_TYPES = ("LUMACLIENT",) + RETURN_NAMES = ("client",) + FUNCTION = "run" + CATEGORY = "LumaAI" + + def run(self, api_key): + """ + Create a LumaAI client with the provided API key. + """ + api_key = api_key if api_key != "" else os.environ.get("LUMAAI_API_KEY", "") + + if api_key == "": + raise ValueError("API Key is required") + + client = LumaAI( + auth_token=api_key, + ) + + return (client,) + + +class Text2Video: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("LUMACLIENT", {"forceInput": True}), + "prompt": ("STRING", {"multiline": True, "default": ""}), + "loop": ("BOOLEAN", {"default": False}), + "aspect_ratio": (["9:16", "3:4", "1:1", "4:3", "16:9", "21:9"],), + "save": ("BOOLEAN", {"default": True}), + }, + "optional": {"filename": ("STRING", {"default": ""})}, + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("video_url", "generation_id") + OUTPUT_NODE = True + FUNCTION = "run" + CATEGORY = "LumaAI" + + def run(self, client, prompt, loop, aspect_ratio, save, filename): + """ + Generate a video from a text prompt. + """ + if prompt == "": + raise ValueError("Prompt is required") + + generation = client.generations.create( + prompt=prompt, loop=loop, aspect_ratio=aspect_ratio + ) + generation_id = generation.id + completed = False + while not completed: + generation = client.generations.get(id=generation_id) + if generation.state == "completed": + completed = True + elif generation.state == "failed": + raise ValueError(f"Generation failed: {generation.failure_reason}") + time.sleep(3) + + video_url = generation.assets.video + if save: + directory, filename = parse_filename(filename) + if filename == "": + filename = generation_id + download_file(video_url, os.path.join(self.output_dir, directory, filename + ".mp4")) + + return { + "ui": {"text": [generation_id]}, + "result": ( + video_url, + generation_id, + ), + } + + +class Image2Video: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("LUMACLIENT", {"forceInput": True}), + "prompt": ("STRING", {"multiline": True, "default": ""}), + "loop": ("BOOLEAN", {"default": False}), + "save": ("BOOLEAN", {"default": True}), + }, + "optional": { + "init_image_url": ("STRING", {"default": "", "forceInput": True}), + "final_image_url": ("STRING", {"default": "", "forceInput": True}), + "filename": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("video_url", "generation_id") + FUNCTION = "run" + CATEGORY = "LumaAI" + + def run( + self, + client, + prompt, + loop, + save, + init_image_url="", + final_image_url="", + filename="", + ): + """ + Generate a video from an image prompt. + """ + if init_image_url == "" and final_image_url == "": + raise ValueError("At least one image URL is required") + + keyframes = {} + if init_image_url != "": + keyframes["frame0"] = {"type": "image", "url": init_image_url} + if final_image_url != "": + keyframes["frame1"] = {"type": "image", "url": final_image_url} + + generation = client.generations.create( + prompt=prompt, loop=loop, keyframes=keyframes + ) + generation_id = generation.id + completed = False + while not completed: + generation = client.generations.get(id=generation_id) + if generation.state == "completed": + completed = True + elif generation.state == "failed": + raise ValueError(f"Generation failed: {generation.failure_reason}") + time.sleep(3) + + video_url = generation.assets.video + if save: + directory, filename = parse_filename(filename) + if filename == "": + filename = generation_id + download_file(video_url, os.path.join(self.output_dir, directory, filename + ".mp4")) + + return { + "ui": {"text": [generation_id]}, + "result": ( + video_url, + generation_id, + ), + } + + +class InterpolateGenerations: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("LUMACLIENT", {"forceInput": True}), + "prompt": ("STRING", {"multiline": True, "default": ""}), + "save": ("BOOLEAN", {"default": True}), + "generation_id_1": ("STRING", {"default": "", "forceInput": True}), + "generation_id_2": ("STRING", {"default": "", "forceInput": True}), + }, + "optional": {"filename": ("STRING", {"default": ""})}, + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("video_url", "generation_id") + FUNCTION = "run" + CATEGORY = "LumaAI" + + def run( + self, + client, + prompt, + loop, + aspect_ratio, + save, + generation_id_1, + generation_id_2, + filename="", + ): + """ + Generate a video by interpolating between two existing generations. + """ + if not generation_id_1 or not generation_id_2: + raise ValueError("Both generation IDs are required") + + generation = client.generations.create( + prompt=prompt, + loop=loop, + aspect_ratio=aspect_ratio, + keyframes={ + "frame0": {"type": "generation", "id": generation_id_1}, + "frame1": {"type": "generation", "id": generation_id_2}, + }, + ) + generation_id = generation.id + completed = False + while not completed: + generation = client.generations.get(id=generation_id) + if generation.state == "completed": + completed = True + elif generation.state == "failed": + raise ValueError(f"Generation failed: {generation.failure_reason}") + time.sleep(3) + + video_url = generation.assets.video + if save: + directory, filename = parse_filename(filename) + if filename == "": + filename = generation_id + download_file(video_url, os.path.join(self.output_dir, directory, filename + ".mp4")) + + return { + "ui": {"text": [generation_id]}, + "result": ( + video_url, + generation_id, + ), + } + + +class ExtendGeneration: + def __init__(self): + self.output_dir = folder_paths.get_output_directory() + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "client": ("LUMACLIENT", {"forceInput": True}), + "prompt": ("STRING", {"multiline": True, "default": ""}), + "save": ("BOOLEAN", {"default": True}), + }, + "optional": { + "init_image_url": ("STRING", {"default": "", "forceInput": True}), + "final_image_url": ("STRING", {"default": "", "forceInput": True}), + "init_generation_id": ("STRING", {"default": "", "forceInput": True}), + "final_generation_id": ("STRING", {"default": "", "forceInput": True}), + "filename": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("video_url", "generation_id") + FUNCTION = "run" + CATEGORY = "LumaAI" + + def run( + self, + client, + prompt, + save, + init_image_url="", + final_image_url="", + init_generation_id="", + final_generation_id="", + filename="", + ): + """ + Generate a video by extending from an image to an existing generation. + """ + if not init_generation_id and not final_generation_id: + raise ValueError("You must provide at least one generation id") + if init_image_url and init_generation_id: + raise ValueError( + "You cannot provide both an init image and a init generation" + ) + if final_image_url and final_generation_id: + raise ValueError( + "You cannot provide both a final image and a final generation" + ) + + keyframes = {} + if init_image_url != "": + keyframes["frame0"] = {"type": "image", "url": init_image_url} + if final_image_url != "": + keyframes["frame1"] = {"type": "image", "url": final_image_url} + if init_generation_id != "": + keyframes["frame0"] = {"type": "generation", "id": init_generation_id} + if final_generation_id != "": + keyframes["frame1"] = {"type": "generation", "id": final_generation_id} + + generation = client.generations.create(prompt=prompt, keyframes=keyframes) + new_generation_id = generation.id + completed = False + while not completed: + generation = client.generations.get(id=new_generation_id) + if generation.state == "completed": + completed = True + elif generation.state == "failed": + raise ValueError(f"Generation failed: {generation.failure_reason}") + time.sleep(3) + + video_url = generation.assets.video + if save: + directory, filename = parse_filename(filename) + if filename == "": + filename = new_generation_id + download_file(video_url, os.path.join(self.output_dir, directory, filename + ".mp4")) + + return { + "ui": {"text": [new_generation_id]}, + "result": ( + video_url, + new_generation_id, + ), + } + + +class PreviewVideo: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "video_url": ("STRING", {"forceInput": True}), + } + } + + OUTPUT_NODE = True + FUNCTION = "run" + CATEGORY = "LumaAI" + RETURN_TYPES = () + + def run(self, video_url): + return {"ui": {"video_url": [video_url]}} diff --git a/py/nodes.py b/py/nodes.py new file mode 100644 index 0000000..4f95b1d --- /dev/null +++ b/py/nodes.py @@ -0,0 +1,29 @@ +from .lumaai_api_node import ( + LumaAIClient, + Text2Video, + Image2Video, + InterpolateGenerations, + ExtendGeneration, + PreviewVideo, +) +from .imgbb_node import ImgBBUpload + +NODE_CLASS_MAPPINGS = { + "LumaAIClient": LumaAIClient, + "ImgBBUpload": ImgBBUpload, + "LumaText2Video": Text2Video, + "LumaImage2Video": Image2Video, + "LumaInterpolateGenerations": InterpolateGenerations, + "LumaExtendGeneration": ExtendGeneration, + "LumaPreviewVideo": PreviewVideo, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "LumaAIClient": "LumaAI Client", + "ImgBBUpload": "ImgBB Upload", + "LumaText2Video": "Text to Video", + "LumaImage2Video": "Image to Video", + "LumaInterpolateGenerations": "Interpolate Generations", + "LumaExtendGeneration": "Extend Generation", + "LumaPreviewVideo": "LumaAI Preview Video", +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6738c78 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "comfyui-lumaai-api" +description = "This custom node allows you to use the LumaAI API." +version = "1.0.1" +license = {file = "LICENSE"} +dependencies = ["lumaai", "requests", "numpy", "Pillow"] + +[project.urls] +Repository = "https://github.com/lumalabs/ComfyUI-LumaAI-API" +# Used by Comfy Registry https://comfyregistry.org + +[tool.comfy] +PublisherId = "lumaai" +DisplayName = "ComfyUI-LumaAI-API" +Icon = "https://cdn-luma.com/public/lumalabs.ai/images/logo.png" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..57c832a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +lumaai +requests +numpy +Pillow \ No newline at end of file diff --git a/web/js/previewVideo.js b/web/js/previewVideo.js new file mode 100644 index 0000000..17b13a0 --- /dev/null +++ b/web/js/previewVideo.js @@ -0,0 +1,154 @@ +// Based on https://github.com/ArtVentureX/comfyui-animatediff/blob/main/web/js/vid_preview.js +import { app, ANIM_PREVIEW_WIDGET } from '../../../scripts/app.js'; +import { createImageHost } from "../../../scripts/ui/imagePreview.js" + +const BASE_SIZE = 768; + +function setVideoDimensions(videoElement, width, height) { + videoElement.style.width = `${width}px`; + videoElement.style.height = `${height}px`; +} + +// Resize video maintaining aspect ratio +export function resizeVideoAspectRatio(videoElement, maxWidth, maxHeight) { + const aspectRatio = videoElement.videoWidth / videoElement.videoHeight; + let newWidth, newHeight; + + // Check which dimension is the limiting factor + if (videoElement.videoWidth / maxWidth > videoElement.videoHeight / maxHeight) { + // Width is the limiting factor + newWidth = maxWidth; + newHeight = newWidth / aspectRatio; + } else { + // Height is the limiting factor + newHeight = maxHeight; + newWidth = newHeight * aspectRatio; + } + + setVideoDimensions(videoElement, newWidth, newHeight); +} + +export function chainCallback(object, property, callback) { + if (object == undefined) { + //This should not happen. + console.error("Tried to add callback to non-existant object"); + return; + } + if (property in object) { + const callback_orig = object[property]; + object[property] = function () { + const r = callback_orig.apply(this, arguments); + callback.apply(this, arguments); + return r; + }; + } else { + object[property] = callback; + } +}; + +export function addVideoPreview(nodeType, options = {}) { + const createVideoNode = (url) => { + return new Promise((cb) => { + const videoEl = document.createElement('video'); + videoEl.addEventListener('loadedmetadata', () => { + videoEl.controls = false; + videoEl.loop = true; + videoEl.muted = true; + resizeVideoAspectRatio(videoEl, BASE_SIZE, BASE_SIZE); + cb(videoEl); + }); + videoEl.addEventListener('error', () => { + cb(); + }); + videoEl.src = url; + }); + }; + + nodeType.prototype.onDrawBackground = function (ctx) { + if (this.flags.collapsed) return; + + let imageURLs = this.images ?? []; + let imagesChanged = false; + + if (JSON.stringify(this.displayingImages) !== JSON.stringify(imageURLs)) { + this.displayingImages = imageURLs; + imagesChanged = true; + } + + if (!imagesChanged) { + return; + } + + if (!imageURLs.length) { + this.imgs = null; + this.animatedImages = false; + return; + } + + const promises = imageURLs.map((url) => { + return createVideoNode(url); + }); + + Promise.all(promises) + .then((imgs) => { + this.imgs = imgs.filter(Boolean); + }) + .then(() => { + if (!this.imgs.length) return; + + this.animatedImages = true; + const widgetIdx = this.widgets?.findIndex((w) => w.name === ANIM_PREVIEW_WIDGET); + + // Set node size to 1024x1024 + this.size[0] = BASE_SIZE; + this.size[1] = BASE_SIZE; + + if (widgetIdx > -1) { + // Replace content + const widget = this.widgets[widgetIdx]; + widget.options.host.updateImages(this.imgs); + } else { + const host = createImageHost(this); + const widget = this.addDOMWidget(ANIM_PREVIEW_WIDGET, 'img', host.el, { + host, + getHeight: host.getHeight, + onDraw: host.onDraw, + hideOnZoom: false, + }); + widget.serializeValue = () => ({ + height: BASE_SIZE, + }); + + widget.options.host.updateImages(this.imgs); + } + + this.imgs.forEach((img) => { + if (img instanceof HTMLVideoElement) { + img.muted = true; + img.autoplay = true; + img.play(); + } + }); + + // Force canvas update + this.setDirtyCanvas(true, true); + }); + }; + + chainCallback(nodeType.prototype, "onExecuted", function (message) { + if (message?.video_url) { + this.images = message?.video_url; + this.setDirtyCanvas(true); + } + }); +} + +app.registerExtension({ + name: "VideoPreview", + async beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData.name !== "LumaPreviewVideo") { + return; + } + addVideoPreview(nodeType); + }, +}); \ No newline at end of file diff --git a/web/js/showGenerationId.js b/web/js/showGenerationId.js new file mode 100644 index 0000000..ba94517 --- /dev/null +++ b/web/js/showGenerationId.js @@ -0,0 +1,52 @@ +//Based on https://github.com/pythongosssss/ComfyUI-Custom-Scripts/blob/main/web/js/showText.js + +import { app } from "../../../scripts/app.js"; +import { ComfyWidgets } from "../../../scripts/widgets.js"; + +// Displays input text on a node +app.registerExtension({ + name: "lumaai.showgenerationid", + async beforeRegisterNodeDef(nodeType, nodeData, app) { + if (nodeData.name == "LumaText2Video" || nodeData.name == "LumaImage2Video" || nodeData.name == "LumaInterpolateGenerations" || nodeData.name == "LumaExtendGeneration") { + function populate(text) { + const v = [...text]; + if (this.widgets) { + if (this.widgets[this.widgets.length - 1].name == "filename") { + const w = ComfyWidgets["STRING"](this, "gen_output", ["STRING", { multiline: true }], app).widget; + w.inputEl.readOnly = true; + w.inputEl.style.opacity = 0.6; + w.value = v[0]; + this.widgets.push(w); + } else { + this.widgets[this.widgets.length - 1].value = v[0]; + } + } + + requestAnimationFrame(() => { + const sz = this.computeSize(); + if (sz[0] < this.size[0]) { + sz[0] = this.size[0]; + } + if (sz[1] < this.size[1]) { + sz[1] = this.size[1]; + } + this.onResize?.(sz); + app.graph.setDirtyCanvas(true, false); + }); + } + + // When the node is executed we will be sent the input text, display this in the widget + const onExecuted = nodeType.prototype.onExecuted; + nodeType.prototype.onExecuted = function (message) { + onExecuted?.apply(this, arguments); + populate.call(this, message.text); + }; + + const onExecutionStart = nodeType.prototype.onExecutionStart; + nodeType.prototype.onExecutionStart = function () { + onExecutionStart?.apply(this, arguments); + populate.call(this, ["generating..."]); + }; + } + }, +}); \ No newline at end of file diff --git a/workflows/ExtendingGeneration.json b/workflows/ExtendingGeneration.json new file mode 100644 index 0000000..01711c1 --- /dev/null +++ b/workflows/ExtendingGeneration.json @@ -0,0 +1,108 @@ +{ + "22": { + "inputs": { + "api_key": "", + "expire": false, + "expiration_time": 60, + "image": [ + "23", + 0 + ] + }, + "class_type": "ImgBBUpload", + "_meta": { + "title": "ImgBB Upload" + } + }, + "23": { + "inputs": { + "image": "IMG_9655.JPG", + "upload": "image" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "25": { + "inputs": { + "api_key": "" + }, + "class_type": "LumaAIClient", + "_meta": { + "title": "LumaAI Client" + } + }, + "26": { + "inputs": { + "video_url": [ + "28", + 0 + ], + "$$comfy_animation_preview": { + "height": 768 + } + }, + "class_type": "LumaPreviewVideo", + "_meta": { + "title": "LumaAI Preview Video" + } + }, + "28": { + "inputs": { + "prompt": "", + "loop": false, + "aspect_ratio": "9:16", + "save": true, + "init_image_url": [ + "22", + 0 + ], + "filename": "", + "gen_output": "generating...", + "client": [ + "25", + 0 + ] + }, + "class_type": "LumaImage2Video", + "_meta": { + "title": "Image to Video" + } + }, + "29": { + "inputs": { + "prompt": " ", + "save": true, + "init_generation_id": [ + "28", + 1 + ], + "filename": "", + "gen_output": "generating...", + "client": [ + "25", + 0 + ] + }, + "class_type": "LumaExtendGeneration", + "_meta": { + "title": "Extend Generation" + } + }, + "31": { + "inputs": { + "video_url": [ + "29", + 0 + ], + "$$comfy_animation_preview": { + "height": 768 + } + }, + "class_type": "LumaPreviewVideo", + "_meta": { + "title": "LumaAI Preview Video" + } + } +} \ No newline at end of file diff --git a/workflows/Image2Video.json b/workflows/Image2Video.json new file mode 100644 index 0000000..7a2b82c --- /dev/null +++ b/workflows/Image2Video.json @@ -0,0 +1,263 @@ +{ + "last_node_id": 28, + "last_link_id": 17, + "nodes": [ + { + "id": 23, + "type": "LoadImage", + "pos": [ + 18, + 590 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 10 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "IMG_9655.JPG", + "image" + ] + }, + { + "id": 25, + "type": "LumaAIClient", + "pos": [ + 30, + 344 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LumaAIClient" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 28, + "type": "LumaImage2Video", + "pos": [ + 727, + 344 + ], + "size": [ + 382.09642410650736, + 212 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "link": 16 + }, + { + "name": "init_image_url", + "type": "STRING", + "link": 15, + "widget": { + "name": "init_image_url" + } + }, + { + "name": "final_image_url", + "type": "STRING", + "link": null, + "widget": { + "name": "final_image_url" + } + } + ], + "outputs": [ + { + "name": "video_url", + "type": "STRING", + "links": [ + 17 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "generation_id", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LumaImage2Video" + }, + "widgets_values": [ + "", + false, + "9:16", + true, + "", + "", + "" + ] + }, + { + "id": 26, + "type": "LumaPreviewVideo", + "pos": [ + 1228, + 342 + ], + "size": [ + 312.26042261370117, + 264.63415629022575 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "video_url", + "type": "STRING", + "link": 17, + "widget": { + "name": "video_url" + }, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LumaPreviewVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 22, + "type": "ImgBBUpload", + "pos": [ + 363, + 589 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 10 + } + ], + "outputs": [ + { + "name": "image_url", + "type": "STRING", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "ImgBBUpload" + }, + "widgets_values": [ + "", + false, + 60 + ] + } + ], + "links": [ + [ + 10, + 23, + 0, + 22, + 0, + "IMAGE" + ], + [ + 15, + 22, + 0, + 28, + 1, + "STRING" + ], + [ + 16, + 25, + 0, + 28, + 0, + "LUMACLIENT" + ], + [ + 17, + 28, + 0, + 26, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6115909044841499, + "offset": [ + 493.78857374805784, + -7.162115379659536 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/workflows/Text2Video.json b/workflows/Text2Video.json new file mode 100644 index 0000000..f5c2979 --- /dev/null +++ b/workflows/Text2Video.json @@ -0,0 +1,149 @@ +{ + "last_node_id": 30, + "last_link_id": 16, + "nodes": [ + { + "id": 30, + "type": "LumaPreviewVideo", + "pos": [ + 1033, + 378 + ], + "size": { + "0": 768, + "1": 768 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "video_url", + "type": "STRING", + "link": 16, + "widget": { + "name": "video_url" + } + } + ], + "properties": { + "Node name for S&R": "LumaPreviewVideo" + }, + "widgets_values": [ + "", + null + ] + }, + { + "id": 28, + "type": "LumaText2Video", + "pos": [ + 526, + 378 + ], + "size": [ + 400.22849849128625, + 198.76139540935253 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "link": 15 + } + ], + "outputs": [ + { + "name": "video_url", + "type": "STRING", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "generation_id", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LumaText2Video" + }, + "widgets_values": [ + "Extreme closeup footage of a young sailor woman with a concerned expression during a rainstorm.", + true, + "9:16", + false, + "" + ] + }, + { + "id": 29, + "type": "LumaAIClient", + "pos": [ + 124, + 375 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "client", + "type": "LUMACLIENT", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LumaAIClient" + }, + "widgets_values": [ + "" + ] + } + ], + "links": [ + [ + 15, + 29, + 0, + 28, + 0, + "LUMACLIENT" + ], + [ + 16, + 28, + 0, + 30, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8954302432552445, + "offset": [ + -56.377797259191, + -151.59915874422396 + ] + } + }, + "version": 0.4 +} \ No newline at end of file