ComfyUI version LumaAPI

2024-06-17 00:00:58 +08:00
commit d967ec7f61
10 changed files with 681 additions and 0 deletions
@@ -0,0 +1,23 @@
+## Unofficial Luma API-ComfyUI version
+
+**WARNING:** This project is for learning purpose only!
+
+### Installation
+
+- cd custom_nodes
+- git clone https://github.com/superyoman/comfyui_lumaAPI.git
+- If necessary: pip install -r requirements.txt
+- Restart ComfyUI
+
+### Info
+
+- Login https://lumalabs.ai/.
+- Use Chrome or other browsers to inspect the network requests (F12 -> XHR).
+- Copy the cookie to key.json
+
+![1](tutorial.png)
+- use image_generation.json (in workflow) for Luma image2video generation
+- use image_generation.json for Luma image2video generation
+- Get the result id, record saved in output/luma video/record_id.txt
+- use check_result.json to get the video link if Luma generation finished,video will be downloaded automatically if *download* is true.
+- Video will be saved in *output/luma video* folder
@@ -0,0 +1,2 @@
+from .luma import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS
+__all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS']
@@ -0,0 +1,121 @@
+import requests
+import time
+
+global lumalabs_internal_url,COMMON_HEADERS
+
+lumalabs_internal_url = 'https://internal-api.virginia.labs.lumalabs.ai/api/photon/v1/generations/'
+COMMON_HEADERS = {
+    "Content-Type": "application/json",
+    "User-Agent": "Mozilla/5.0(Macintosh;U;IntelMacOSX10_6_8;en-us)AppleWebKit/534.50(KHTML,likeGecko)Version/5.1Safari/534.50",
+    "Referer": "https://lumalabs.ai/",
+    "Origin": "https://lumalabs.ai",
+}
+
+def get_video_response(ids):
+    api_url = f"{lumalabs_internal_url}/{ids}"
+    response = requests.get(url = api_url,headers = COMMON_HEADERS)
+    return response.json()
+
+def get_video(ids):
+    res = get_video_response(ids)
+    if "id" in res:
+        print(type(res))
+        return res['video']
+
+def get_upload_link(Cookie):
+    full_url = f"{lumalabs_internal_url}file_upload"
+    params = {
+        'file_type': 'image',
+        'filename': 'file.jpg'
+    }
+    response = requests.post(full_url, params = params, cookies = Cookie)
+    presigned_url, public_url = response.json()["presigned_url"], response.json()["public_url"]
+    print(presigned_url)
+    print(public_url)
+    return presigned_url, public_url
+
+def upload_file(Cookie, file_path):
+    try:
+        presigned_url, public_url =  get_upload_link(Cookie)
+        time.sleep(1)
+        with open(file_path, 'rb') as file:
+            response = requests.put(presigned_url, data=file,
+                        headers={'Content-Type': "image/*", "Referer": "https://lumalabs.ai/", "origin": "https://lumalabs.ai"})
+
+        if response.status_code == 200:
+            print("Upload successful:", public_url)
+            return public_url
+        else:
+            print("Upload failed.")
+    except Exception as e:
+        print("Upload failed:", e)
+
+
+def upload_file_comfyui(Cookie, binary_image):
+    try:
+        presigned_url, public_url = get_upload_link(Cookie)
+        time.sleep(1)
+        response = requests.put(presigned_url, data=binary_image,
+                        headers={'Content-Type': "image/*", "Referer": "https://lumalabs.ai/", "origin": "https://lumalabs.ai"})
+
+        if response.status_code == 200:
+            print("Upload successful:", public_url)
+            return public_url
+        else:
+            print("Upload failed.")
+    except Exception as e:
+        print("Upload failed:", e)
+
+def DreamMachine_generation(prompt,Cookie,image_file = None,binary_image = None, expand_prompt = True):
+    payload = {
+        "user_prompt": prompt,
+        "aspect_ratio": "16:9",
+        "expand_prompt": expand_prompt
+    }
+    Cookie_dict = {
+        "Cookie":Cookie
+    }
+    if image_file:
+        image_url = upload_file(Cookie_dict, image_file)
+        if image_url.startswith("https:"):
+            payload["image_url"] = image_url
+        else:
+            return "Something Wrong"
+
+    if binary_image:
+        image_url = upload_file_comfyui(Cookie_dict, binary_image)
+        if image_url.startswith("https:"):
+            payload["image_url"] = image_url
+        else:
+            return "Something Wrong"
+
+
+    response = requests.post(lumalabs_internal_url, json=payload, cookies = Cookie_dict, headers=COMMON_HEADERS)
+    response_json = response.json()
+    return response_json
+
+def check_res(Cookie = str):
+    try:
+        url =  "https://internal-api.virginia.labs.lumalabs.ai/api/photon/v1/user/generations/"
+        params = {"offset": "0", "limit": "10"}
+        res = requests.get(url,headers = {"Cookie":Cookie},params = params)
+        return res.json()
+    except Exception as ex:
+        print("Failed to get result: %s"%ex)
+
+def get_signal_video(ids = str,Cookie = str,download = True, output_name = 'output_video.mp4'):
+    res = check_res(Cookie)
+    for i in res:
+        if ids == i['id']:
+            video_url = i["video"]['url']
+            if download:
+                with open(output_name, 'wb') as file:
+                    response = requests.get(video_url, stream=True)
+                    for chunk in response.iter_content(chunk_size=1024):
+                        if chunk:
+                            file.write(chunk)
+            return video_url
+    return "No result"
+
+if __name__ == "__main__":
+    pass
@@ -0,0 +1,3 @@
+{
+    "cookie": "_ga=GA1.1.15044adadaWIiOnsidXNlcl91dWlkIjoiY2QzMTRmY2#RADSFVW5adad$WFWASDFAFAD%ASDAfad552y9v5dbC7xYVxo77WhaAk_xb46W9xrzgEgo; refresh_token=sfsfsfpZW50X2lkIjoiIn0sImV4cCI6MTcxOTA4MTU3NH0.QQiEt3EPvy9v5dbC7xYVxo77WhaAk_xb46W9xrzgEgo; _clck=dfssfsf627; _clsk=1g1u83g%7C1718548919433%7C7%7C0%7Ce.clarity.ms%2Fcollect; _ga856613.0.0.0"
+}
@@ -0,0 +1,121 @@
+import os
+import io
+import json
+import torch
+from . import generator
+from io import BytesIO
+from PIL import Image
+
+global luma_key_path
+lunma_key_path = os.path.dirname(os.path.realpath(__file__))
+
+def get_cookie_key():
+    try:
+        config_path = os.path.join(lunma_key_path, 'key.json')
+        with open(config_path, 'r') as f:
+            config = json.load(f)
+        api_key = config["cookie"]
+    except:
+        print("Error: Cookie is required. Please enter correct cookie in key.json")
+        return ""
+    return api_key
+
+def create_folder(path):
+    try:
+        os.makedirs(path, exist_ok=True)
+    except Exception as e:
+        print(f"Fail to create the folder：{e}")
+
+class LUMA_YoC_generator:
+
+    def __init__(self):
+        self.api_key = get_cookie_key()
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "image": ("IMAGE",),
+                "prompt": ("STRING", {"default": "movie,cinematic", "multiline": True}),
+                "Enhance_prompt": ("BOOLEAN", {"default": True}),
+            }
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("luma_id",)
+    FUNCTION = "LUMA_generation"
+    CATEGORY = "YoC app/LumaAPP"
+
+    def LUMA_generation(self,image, prompt,Enhance_prompt):
+        tensor = image
+        if tensor.dim() == 4 and tensor.shape[0] == 1:
+            tensor = tensor.squeeze(0)  # 去掉批次维度，形状变为[3, H, W]
+
+        # 如果通道在前，需要调整为[H, W, 3]以适应PIL或其他图像库
+        if tensor.shape[0] == 3:
+            tensor = tensor.permute(1, 2, 0)
+        # 确保tensor是浮点类型并且范围在[0, 1]，如果不是，您可能需要先进行归一化或缩放
+        tensor = tensor.clamp(0, 1)  # 确保值在[0, 1]之间
+
+        # 判断图像是否为空白
+        if torch.std(tensor) < 0.01:  # 可调整阈值以适应具体情况
+            print("Empty Image")
+            image_binary = None
+        else:       
+            # 转换为NumPy数组
+            numpy_array = tensor.numpy()
+            buffer = BytesIO()
+            # 转换为PIL图像并保存
+            image = Image.fromarray((numpy_array * 255).astype('uint8'))  # 转换为0-255范围
+            image.save(buffer, format='PNG')  # 您可以选择其他格式，如JPEG
+    
+            # 获取二进制数据
+            image_binary = buffer.getvalue()
+
+        res = generator.DreamMachine_generation(prompt=prompt, binary_image = image_binary,
+                                                Cookie=self.api_key, expand_prompt=Enhance_prompt)
+        res_id = res[0]['id']
+        print("Result id: %s"%res_id)
+        create_folder(os.path.join(os.getcwd(), "ComfyUI/output/luma video"))
+        with open(os.path.join(os.path.join(os.getcwd(), "ComfyUI/output/luma video"),"record_id.txt"), 'a') as file:
+            file.write(res_id + '\n')
+            
+        return (res_id,)
+
+class LUMA_YoC_result:
+
+    def __init__(self):
+        self.api_key = get_cookie_key()
+
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "text": ("STRING", {"default": "Enter your id", "multiline": False}),
+                "download":("BOOLEAN", {"default": True}),
+            }
+        }
+
+    RETURN_TYPES = ("STRING",)
+    RETURN_NAMES = ("video_link",)
+    FUNCTION = "LUMA_result"
+    CATEGORY = "YoC app/LumaAPP"
+
+    def LUMA_result(self,text,download):
+        video_folder = os.path.join(os.getcwd(), "ComfyUI/output/luma video")
+        create_folder(video_folder)
+        res_link = generator.get_signal_video(ids = text, Cookie = self.api_key,
+                        download = download,output_name=os.path.join(video_folder,f"{text}.mp4"))
+        return (res_link,)
+
+NODE_CLASS_MAPPINGS = {
+    "LUMA_API_YoC": LUMA_YoC_generator,
+    "LUMA_API_result_YoC": LUMA_YoC_result,
+
+}
+
+NODE_DISPLAY_NAME_MAPPINGS = {
+    "LUMA_API_YoC": "LUMA_generator @.@",
+    "LUMA_API_result_YoC": "LUMA_YoC_result @.@",
+}
@@ -0,0 +1,2 @@
+requests
+Pillow
@@ -0,0 +1,101 @@
+{
+  "last_node_id": 9,
+  "last_link_id": 4,
+  "nodes": [
+    {
+      "id": 9,
+      "type": "LUMA_API_result_YoC",
+      "pos": [
+        359,
+        -14
+      ],
+      "size": {
+        "0": 315,
+        "1": 82
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "video_link",
+          "type": "STRING",
+          "links": [
+            4
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LUMA_API_result_YoC"
+      },
+      "widgets_values": [
+        "486c63a7-7710-402b-bbd6-fcb6c1ef20fa",
+        true
+      ]
+    },
+    {
+      "id": 6,
+      "type": "ShowText|pysssss",
+      "pos": [
+        921,
+        -12
+      ],
+      "size": {
+        "0": 315,
+        "1": 76
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 4,
+          "widget": {
+            "name": "text"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": null,
+          "shape": 6
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ShowText|pysssss"
+      },
+      "widgets_values": [
+        "",
+        "https://storage.cdn-luma.com/lit_lite_inference_im2vid_v1.0/afa4ceb9-5e3a-4919-a1b9-8616f28ed8a9/watermarked_video0dbd169287fab4b5a966b69f0b5cf8a86.mp4"
+      ]
+    }
+  ],
+  "links": [
+    [
+      4,
+      9,
+      0,
+      6,
+      0,
+      "STRING"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 1.2839025177495011,
+      "offset": [
+        -196.4949846033825,
+        324.45706003508246
+      ]
+    }
+  },
+  "version": 0.4
+}
@@ -0,0 +1,156 @@
+{
+  "last_node_id": 18,
+  "last_link_id": 7,
+  "nodes": [
+    {
+      "id": 6,
+      "type": "ShowText|pysssss",
+      "pos": [
+        1034,
+        -47
+      ],
+      "size": {
+        "0": 435.64239501953125,
+        "1": 149.36404418945312
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 5,
+          "widget": {
+            "name": "text"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": null,
+          "shape": 6,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ShowText|pysssss"
+      },
+      "widgets_values": [
+        "",
+        "No result"
+      ]
+    },
+    {
+      "id": 16,
+      "type": "LUMA_API_YoC",
+      "pos": [
+        424,
+        -100
+      ],
+      "size": {
+        "0": 400,
+        "1": 200
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 7
+        }
+      ],
+      "outputs": [
+        {
+          "name": "luma_id",
+          "type": "STRING",
+          "links": [
+            5
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LUMA_API_YoC"
+      },
+      "widgets_values": [
+        "a tree, movie, cinematic",
+        false
+      ]
+    },
+    {
+      "id": 18,
+      "type": "LoadImage",
+      "pos": [
+        -94,
+        -186
+      ],
+      "size": [
+        315,
+        314
+      ],
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            7
+          ],
+          "shape": 3,
+          "slot_index": 0
+        },
+        {
+          "name": "MASK",
+          "type": "MASK",
+          "links": null,
+          "shape": 3
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LoadImage"
+      },
+      "widgets_values": [
+        "b1.png",
+        "image"
+      ]
+    }
+  ],
+  "links": [
+    [
+      5,
+      16,
+      0,
+      6,
+      0,
+      "STRING"
+    ],
+    [
+      7,
+      18,
+      0,
+      16,
+      0,
+      "IMAGE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 1.061076460950001,
+      "offset": [
+        188.41497794560198,
+        506.01070055609375
+      ]
+    }
+  },
+  "version": 0.4
+}
@@ -0,0 +1,152 @@
+{
+  "last_node_id": 17,
+  "last_link_id": 6,
+  "nodes": [
+    {
+      "id": 6,
+      "type": "ShowText|pysssss",
+      "pos": [
+        1034,
+        -47
+      ],
+      "size": {
+        "0": 435.64239501953125,
+        "1": 149.36404418945312
+      },
+      "flags": {},
+      "order": 2,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "text",
+          "type": "STRING",
+          "link": 5,
+          "widget": {
+            "name": "text"
+          }
+        }
+      ],
+      "outputs": [
+        {
+          "name": "STRING",
+          "type": "STRING",
+          "links": null,
+          "shape": 6,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "ShowText|pysssss"
+      },
+      "widgets_values": [
+        "",
+        "No result"
+      ]
+    },
+    {
+      "id": 17,
+      "type": "EmptyImage",
+      "pos": [
+        -59.833016955203675,
+        -149.61822598071518
+      ],
+      "size": {
+        "0": 315,
+        "1": 130
+      },
+      "flags": {},
+      "order": 0,
+      "mode": 0,
+      "outputs": [
+        {
+          "name": "IMAGE",
+          "type": "IMAGE",
+          "links": [
+            6
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "EmptyImage"
+      },
+      "widgets_values": [
+        512,
+        512,
+        1,
+        0
+      ]
+    },
+    {
+      "id": 16,
+      "type": "LUMA_API_YoC",
+      "pos": [
+        424,
+        -100
+      ],
+      "size": {
+        "0": 400,
+        "1": 200
+      },
+      "flags": {},
+      "order": 1,
+      "mode": 0,
+      "inputs": [
+        {
+          "name": "image",
+          "type": "IMAGE",
+          "link": 6
+        }
+      ],
+      "outputs": [
+        {
+          "name": "luma_id",
+          "type": "STRING",
+          "links": [
+            5
+          ],
+          "shape": 3,
+          "slot_index": 0
+        }
+      ],
+      "properties": {
+        "Node name for S&R": "LUMA_API_YoC"
+      },
+      "widgets_values": [
+        "a tree, movie, cinematic",
+        false
+      ]
+    }
+  ],
+  "links": [
+    [
+      5,
+      16,
+      0,
+      6,
+      0,
+      "STRING"
+    ],
+    [
+      6,
+      17,
+      0,
+      16,
+      0,
+      "IMAGE"
+    ]
+  ],
+  "groups": [],
+  "config": {},
+  "extra": {
+    "ds": {
+      "scale": 1.061076460950001,
+      "offset": [
+        188.41497794560198,
+        506.01070055609375
+      ]
+    }
+  },
+  "version": 0.4
+}