Add video generation functionality and FLUX.1 Krea model support
- Add Runware Text to Video and Image to Video nodes - Add FLUX.1 Krea model to all search modules
This commit is contained in:
Sirshendu Ganguly
@@ -122,6 +122,10 @@ Here’s a breakdown of the amazing nodes included in this integration:
|
||||
- **Runware IPAdapter**: Use reference images to guide the style and content of generated images. 🖌️
|
||||
- **Runware IPAdapters Combine**: Combine multiple IP-Adapter inputs for sophisticated image conditioning. 🔗
|
||||
|
||||
|
||||
- **Runware Text to Video**: Generate videos from text prompts using advanced AI models. 🎥
|
||||
- **Runware Image to Video**: Create videos from reference images with optional text guidance. 🖼️➡️🎥
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Support & Community
|
||||
|
||||
@@ -25,6 +25,8 @@ from .modules.ipAdapter import ipAdapter
|
||||
from .modules.ipAdapterCombine import ipAdapterCombine
|
||||
from .modules.vaeSearch import vaeSearch
|
||||
from .modules.referenceImages import referenceImages
|
||||
from .modules.videoInference import txt2vid
|
||||
from .modules.videoModelSearch import videoModelSearch
|
||||
|
||||
RUNWARE_COMFYUI_VERSION = "1.2.0 Beta"
|
||||
|
||||
@@ -64,6 +66,8 @@ NODE_CLASS_MAPPINGS = {
|
||||
"Runware IPAdapter": ipAdapter,
|
||||
"Runware IPAdapters Combine": ipAdapterCombine,
|
||||
"Runware Reference Images": referenceImages,
|
||||
"Runware Video Inference": txt2vid,
|
||||
"Runware Video Model Search": videoModelSearch,
|
||||
}
|
||||
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
@@ -72,6 +76,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"Runware Embedding Search": "Runware Embedding",
|
||||
"Runware VAE Search": "Runware VAE",
|
||||
"Runware Multi Inference": "Runware Multi Inference [BETA]",
|
||||
"Runware Video Model Search": "Runware Video Model",
|
||||
}
|
||||
|
||||
WEB_DIRECTORY = "./clientlibs"
|
||||
|
||||
@@ -13,6 +13,7 @@ const DEFAULT_MODELS_ARCH_LIST = {
|
||||
"All": "all",
|
||||
"FLUX.1-Schnell": "flux1s",
|
||||
"FLUX.1-Dev": "flux1d",
|
||||
"FLUX.1 Krea": "flux1k",
|
||||
"Pony": "pony",
|
||||
"SD 1.5": "sd1x",
|
||||
"SD 1.5 Hyper": "sdhyper",
|
||||
@@ -73,6 +74,8 @@ const RUNWARE_NODE_TYPES = {
|
||||
EMBEDDINGCOMBINE: "Runware Embedding Combine",
|
||||
VAE: "Runware VAE Search",
|
||||
REFERENCEIMAGES: "Runware Reference Images",
|
||||
VIDEOINFERENCE: "Runware Video Inference",
|
||||
VIDEOMODELSEARCH: "Runware Video Model Search",
|
||||
};
|
||||
|
||||
const RUNWARE_NODE_PROPS = {
|
||||
@@ -186,6 +189,15 @@ const RUNWARE_NODE_PROPS = {
|
||||
bgColor: DEFAULT_BGCOLOR,
|
||||
colorModeOnly: true,
|
||||
},
|
||||
[RUNWARE_NODE_TYPES.VIDEOINFERENCE]: {
|
||||
bgColor: DEFAULT_BGCOLOR,
|
||||
liveDimensions: true,
|
||||
promptEnhancer: true,
|
||||
},
|
||||
[RUNWARE_NODE_TYPES.VIDEOMODELSEARCH]: {
|
||||
bgColor: DEFAULT_BGCOLOR,
|
||||
liveSearch: true,
|
||||
},
|
||||
};
|
||||
|
||||
export {
|
||||
|
||||
@@ -15,6 +15,7 @@ class controlNet:
|
||||
"All",
|
||||
"FLUX.1-Schnell",
|
||||
"FLUX.1-Dev",
|
||||
"FLUX.1 Krea",
|
||||
"Pony",
|
||||
"SD 1.5",
|
||||
"SD 1.5 Hyper",
|
||||
|
||||
@@ -10,6 +10,7 @@ class embeddingSearch:
|
||||
"All",
|
||||
"FLUX.1-Schnell",
|
||||
"FLUX.1-Dev",
|
||||
"FLUX.1 Krea",
|
||||
"Pony",
|
||||
"SD 1.5",
|
||||
"SD 1.5 Hyper",
|
||||
|
||||
@@ -10,6 +10,7 @@ class loraSearch:
|
||||
"All",
|
||||
"FLUX.1-Schnell",
|
||||
"FLUX.1-Dev",
|
||||
"FLUX.1 Krea",
|
||||
"Pony",
|
||||
"SD 1.5",
|
||||
"SD 1.5 Hyper",
|
||||
|
||||
@@ -10,6 +10,7 @@ class modelSearch:
|
||||
"All",
|
||||
"FLUX.1-Schnell",
|
||||
"FLUX.1-Dev",
|
||||
"FLUX.1 Krea",
|
||||
"Pony",
|
||||
"SD 1.5",
|
||||
"SD 1.5 Hyper",
|
||||
@@ -37,6 +38,7 @@ class modelSearch:
|
||||
"rundiffusion:130@100 (Juggernaut Pro Flux by RunDiffusion)",
|
||||
"runware:100@1 (Flux Schnell)",
|
||||
"runware:101@1 (Flux Dev)",
|
||||
"runware:107@1 (FLUX.1 Krea)",
|
||||
"runware:5@1 (SD3)",
|
||||
"civitai:4384@128713 (SDXL 1.5 DreamShaper)",
|
||||
"civitai:43331@176425 (SD 1.5 majicMIX realistic 麦橘写实)",
|
||||
|
||||
+570
-414
@@ -1,414 +1,570 @@
|
||||
from comfy.model_management import InterruptProcessingException
|
||||
from requests.adapters import HTTPAdapter
|
||||
from datetime import datetime, timedelta
|
||||
from server import PromptServer
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
import requests
|
||||
import hashlib
|
||||
import asyncio
|
||||
import random
|
||||
import base64
|
||||
import torch
|
||||
import uuid
|
||||
import time
|
||||
import json
|
||||
import os
|
||||
import io
|
||||
import threading
|
||||
|
||||
load_dotenv()
|
||||
|
||||
BASEFOLDER = Path(__file__).parent.parent.parent
|
||||
IMAGE_CACHE_FILE = BASEFOLDER / "imagesCache.json"
|
||||
|
||||
if not IMAGE_CACHE_FILE.exists():
|
||||
with open(IMAGE_CACHE_FILE, "w") as f:
|
||||
print("[Runware] Initializing images cache...")
|
||||
json.dump({}, f)
|
||||
|
||||
RUNWARE_REMBG_OUTPUT_FORMATS = {
|
||||
"outputFormat": (
|
||||
["WEBP", "PNG"],
|
||||
{"default": "WEBP", "tooltip": "Choose the output image format."},
|
||||
)
|
||||
}
|
||||
|
||||
MAX_RETRIES = 4
|
||||
RETRY_COOLDOWNS = [1, 2, 5, 10]
|
||||
|
||||
session = requests.Session()
|
||||
adapter = HTTPAdapter(pool_connections=10, pool_maxsize=10)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
|
||||
def generalRequestWrapper(recaller, *args, **kwargs):
|
||||
for attempt in range(MAX_RETRIES + 1):
|
||||
try:
|
||||
return recaller(*args, **kwargs)
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.SSLError) as e:
|
||||
if attempt == MAX_RETRIES:
|
||||
raise
|
||||
else:
|
||||
cooldown = RETRY_COOLDOWNS[attempt]
|
||||
print(f"[Runware] Error API Request Failed! Retrying in {cooldown} seconds... (Attempt {attempt+1}/{MAX_RETRIES})")
|
||||
time.sleep(cooldown)
|
||||
continue
|
||||
return False
|
||||
|
||||
def getAPIKey():
|
||||
apiKey = os.getenv("RUNWARE_API_KEY")
|
||||
if apiKey and isinstance(apiKey, str) and len(apiKey) > 24:
|
||||
return apiKey
|
||||
return False
|
||||
|
||||
def getTimeout():
|
||||
timeout = os.getenv("RUNWARE_TIMEOUT")
|
||||
if timeout and timeout.isdigit():
|
||||
return int(timeout)
|
||||
else:
|
||||
timeout = 90
|
||||
os.environ["RUNWARE_TIMEOUT"] = str(timeout)
|
||||
return timeout
|
||||
|
||||
def getOutputQuality():
|
||||
output_quality = os.getenv("RUNWARE_OUTPUT_QUALITY")
|
||||
if output_quality and isinstance(output_quality, str) and output_quality.isdigit():
|
||||
return int(output_quality)
|
||||
else:
|
||||
output_quality = 95
|
||||
os.environ["RUNWARE_OUTPUT_QUALITY"] = str(output_quality)
|
||||
return output_quality
|
||||
|
||||
def getOutputFormat():
|
||||
output_format = os.getenv("RUNWARE_OUTPUT_FORMAT")
|
||||
if output_format and isinstance(output_format, str):
|
||||
return output_format
|
||||
else:
|
||||
output_format = "WEBP"
|
||||
os.environ["RUNWARE_OUTPUT_FORMAT"] = output_format
|
||||
return output_format
|
||||
|
||||
def getEnableImagesCaching():
|
||||
enable_images_caching = os.getenv("RUNWARE_ENABLE_IMAGES_CACHING")
|
||||
if enable_images_caching and enable_images_caching.lower() in ["true", "false"]:
|
||||
return enable_images_caching.lower() == "true"
|
||||
else:
|
||||
enable_images_caching = True
|
||||
os.environ["RUNWARE_ENABLE_IMAGES_CACHING"] = str(enable_images_caching)
|
||||
return enable_images_caching
|
||||
|
||||
def getMinImageCacheSize():
|
||||
min_image_cache_size = os.getenv("RUNWARE_MIN_IMAGE_CACHE_SIZE")
|
||||
if min_image_cache_size and min_image_cache_size.isdigit():
|
||||
return int(min_image_cache_size)
|
||||
else:
|
||||
min_image_cache_size = 150
|
||||
os.environ["RUNWARE_MIN_IMAGE_CACHE_SIZE"] = str(min_image_cache_size)
|
||||
return min_image_cache_size
|
||||
|
||||
def getCustomEndpoint():
|
||||
custom_endpoint = os.getenv("RUNWARE_CUSTOM_ENDPOINT")
|
||||
if custom_endpoint and isinstance(custom_endpoint, str):
|
||||
return custom_endpoint
|
||||
return "https://api.runware.ai/v1"
|
||||
|
||||
SESSION_TIMEOUT = getTimeout()
|
||||
RUNWARE_API_KEY = getAPIKey()
|
||||
OUTPUT_FORMAT = getOutputFormat()
|
||||
OUTPUT_QUALITY = getOutputQuality()
|
||||
ENABLE_IMAGES_CACHING = getEnableImagesCaching()
|
||||
MIN_IMAGE_CACHE_SIZE = getMinImageCacheSize()
|
||||
RUNWARE_API_BASE_URL = getCustomEndpoint()
|
||||
|
||||
def setEnvKey(keyName, keyValue):
|
||||
comfyNodeRoot = Path(__file__).parent.parent.parent
|
||||
envFilePath = comfyNodeRoot / ".env"
|
||||
if not envFilePath.exists():
|
||||
envFilePath.touch()
|
||||
with open(envFilePath, "r") as f:
|
||||
lines = f.readlines()
|
||||
key_exists = False
|
||||
new_lines = []
|
||||
for line in lines:
|
||||
if line.startswith(f"{keyName}="):
|
||||
key_exists = True
|
||||
new_lines.append(f"{keyName}={keyValue}\n")
|
||||
else:
|
||||
new_lines.append(line)
|
||||
if not key_exists:
|
||||
new_lines.append(f"{keyName}={keyValue}\n")
|
||||
with open(envFilePath, "w") as f:
|
||||
f.writelines(new_lines)
|
||||
return True
|
||||
|
||||
def setAPIKey(apiKey: str):
|
||||
global RUNWARE_API_KEY
|
||||
envSetRes = setEnvKey("RUNWARE_API_KEY", apiKey)
|
||||
if envSetRes:
|
||||
RUNWARE_API_KEY = apiKey
|
||||
os.environ["RUNWARE_API_KEY"] = apiKey
|
||||
return True
|
||||
|
||||
def setTimeout(timeout: int):
|
||||
envSetRes = setEnvKey("RUNWARE_TIMEOUT", str(timeout))
|
||||
if envSetRes:
|
||||
global SESSION_TIMEOUT
|
||||
SESSION_TIMEOUT = timeout
|
||||
os.environ["RUNWARE_TIMEOUT"] = str(timeout)
|
||||
return True
|
||||
|
||||
def setOutputFormat(format: str):
|
||||
envSetRes = setEnvKey("RUNWARE_OUTPUT_FORMAT", format)
|
||||
if envSetRes:
|
||||
global OUTPUT_FORMAT
|
||||
OUTPUT_FORMAT = format
|
||||
os.environ["RUNWARE_OUTPUT_FORMAT"] = format
|
||||
return True
|
||||
|
||||
def setOutputQuality(quality: int):
|
||||
envSetRes = setEnvKey("RUNWARE_OUTPUT_QUALITY", str(quality))
|
||||
if envSetRes:
|
||||
global OUTPUT_QUALITY
|
||||
OUTPUT_QUALITY = quality
|
||||
os.environ["RUNWARE_OUTPUT_QUALITY"] = str(quality)
|
||||
return True
|
||||
|
||||
def setEnableImagesCaching(enabled: bool):
|
||||
envSetRes = setEnvKey("RUNWARE_ENABLE_IMAGES_CACHING", str(enabled))
|
||||
if envSetRes:
|
||||
global ENABLE_IMAGES_CACHING
|
||||
ENABLE_IMAGES_CACHING = enabled
|
||||
os.environ["RUNWARE_ENABLE_IMAGES_CACHING"] = str(enabled)
|
||||
return True
|
||||
|
||||
def setMinImageCacheSize(size: int):
|
||||
envSetRes = setEnvKey("RUNWARE_MIN_IMAGE_CACHE_SIZE", str(size))
|
||||
if envSetRes:
|
||||
global MIN_IMAGE_CACHE_SIZE
|
||||
MIN_IMAGE_CACHE_SIZE = size
|
||||
os.environ["RUNWARE_MIN_IMAGE_CACHE_SIZE"] = str(size)
|
||||
return True
|
||||
|
||||
def genRandSeed(minSeed=1000, maxSeed=9223372036854776000):
|
||||
return random.randint(minSeed, maxSeed)
|
||||
|
||||
def genRandUUID():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
def checkAPIKey(apiKey):
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"Accept-Encoding": "gzip, deflate, br, zstd",
|
||||
}
|
||||
|
||||
genConfig = [
|
||||
{
|
||||
"taskType": "authentication",
|
||||
"apiKey": apiKey,
|
||||
}
|
||||
]
|
||||
|
||||
try:
|
||||
|
||||
def recaller():
|
||||
return session.post(
|
||||
RUNWARE_API_BASE_URL,
|
||||
headers=headers,
|
||||
json=genConfig,
|
||||
timeout=10,
|
||||
allow_redirects=False,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
genResult = generalRequestWrapper(recaller)
|
||||
genResult = genResult.json()
|
||||
if "errors" in genResult:
|
||||
return str(genResult["errors"][0]["message"])
|
||||
else:
|
||||
return True
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
def sendImageCaption(captionText, nodeID):
|
||||
PromptServer.instance.send_sync(
|
||||
"runwareImageCaption",
|
||||
{
|
||||
"success": True,
|
||||
"captionText": captionText,
|
||||
"nodeID": nodeID,
|
||||
},
|
||||
)
|
||||
|
||||
def inferenecRequest(genConfig):
|
||||
global RUNWARE_API_KEY, RUNWARE_API_BASE_URL, SESSION_TIMEOUT
|
||||
RUNWARE_API_KEY = os.getenv("RUNWARE_API_KEY")
|
||||
SESSION_TIMEOUT = int(os.getenv("RUNWARE_TIMEOUT"))
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNWARE_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept-Encoding": "gzip, deflate, br, zstd",
|
||||
}
|
||||
|
||||
try:
|
||||
def recaller():
|
||||
return session.post(
|
||||
RUNWARE_API_BASE_URL,
|
||||
headers=headers,
|
||||
json=genConfig,
|
||||
timeout=SESSION_TIMEOUT,
|
||||
allow_redirects=False,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
genResult = generalRequestWrapper(recaller)
|
||||
try:
|
||||
genResult = genResult.json()
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"[Debugging] Runware JSON Decode Error: {str(e)}")
|
||||
print(f"[Debugging] Runware Response Status Code: {genResult.status_code}")
|
||||
print(f"[Debugging] Runware Response Headers: {genResult.headers}")
|
||||
print(f"[Debugging] Runware Raw Response Content: {genResult.content}")
|
||||
raise Exception("Error: Invalid JSON response from API!")
|
||||
if "errors" in genResult:
|
||||
raise Exception(genResult["errors"][0]["message"])
|
||||
else:
|
||||
return genResult
|
||||
except requests.exceptions.Timeout:
|
||||
raise Exception(
|
||||
f"Error: Request Timed Out After {SESSION_TIMEOUT} Seconds - Please Try Again!"
|
||||
)
|
||||
except requests.exceptions.RequestException:
|
||||
raise Exception("Error: Runware Request Failed!")
|
||||
except Exception as e:
|
||||
if "invalid api key" in str(e).lower():
|
||||
PromptServer.instance.send_sync(
|
||||
"runwareError",
|
||||
{
|
||||
"success": False,
|
||||
"errorMessage": str(e),
|
||||
"errorCode": 401,
|
||||
},
|
||||
)
|
||||
raise InterruptProcessingException()
|
||||
else:
|
||||
raise Exception(f"Error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def uploadImage(imageDataUri):
|
||||
uploadTaskConfig = [
|
||||
{"taskType": "imageUpload", "taskUUID": genRandUUID(), "image": imageDataUri}
|
||||
]
|
||||
|
||||
try:
|
||||
uploadResult = inferenecRequest(uploadTaskConfig)
|
||||
if (
|
||||
uploadResult
|
||||
and "data" in uploadResult
|
||||
and "imageUUID" in uploadResult["data"][0]
|
||||
):
|
||||
imageUUID = uploadResult["data"][0]["imageUUID"]
|
||||
return imageUUID
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
async def uploadAndCacheImage(imgSig: str, imgDataUri: str):
|
||||
try:
|
||||
imageUUID = await uploadImage(imgDataUri)
|
||||
if imageUUID:
|
||||
await imageStoreSet(imgSig, imageUUID)
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
async def imageStoreSet(imgHash: str, imgUUID: str) -> bool:
|
||||
try:
|
||||
with open(IMAGE_CACHE_FILE, "r") as f:
|
||||
cache = json.load(f)
|
||||
expires = (datetime.now() + timedelta(days=30)).isoformat()
|
||||
cache[imgHash] = {"uuid": imgUUID, "expires": expires}
|
||||
with open(IMAGE_CACHE_FILE, "w") as f:
|
||||
json.dump(cache, f, indent=2)
|
||||
return True
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
def imageStoreGet(imgHash: str) -> str | bool:
|
||||
try:
|
||||
with open(IMAGE_CACHE_FILE, "r") as f:
|
||||
cache = json.load(f)
|
||||
if imgHash not in cache:
|
||||
return False
|
||||
entry = cache[imgHash]
|
||||
expires = datetime.fromisoformat(entry["expires"])
|
||||
if datetime.now() > expires:
|
||||
del cache[imgHash]
|
||||
with open(IMAGE_CACHE_FILE, "w") as f:
|
||||
json.dump(cache, f, indent=2)
|
||||
return False
|
||||
return entry["uuid"]
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
def convertTensor2IMG(tensorImage):
|
||||
global ENABLE_IMAGES_CACHING, MIN_IMAGE_CACHE_SIZE
|
||||
ENABLE_IMAGES_CACHING = getEnableImagesCaching()
|
||||
MIN_IMAGE_CACHE_SIZE = int(os.getenv("RUNWARE_MIN_IMAGE_CACHE_SIZE"))
|
||||
|
||||
imageNP = (tensorImage.squeeze().numpy() * 255).astype(np.uint8)
|
||||
imgSig = hashlib.sha256(imageNP.tobytes()).hexdigest()
|
||||
|
||||
if ENABLE_IMAGES_CACHING:
|
||||
imgUUID = imageStoreGet(imgSig)
|
||||
if imgUUID:
|
||||
return imgUUID
|
||||
|
||||
image = Image.fromarray(imageNP)
|
||||
with io.BytesIO() as buffer:
|
||||
image.save(
|
||||
buffer, format="webp", quality=100, subsampling=0, method=6, exact=True
|
||||
)
|
||||
imageRawData = buffer.getvalue()
|
||||
imgBytes = len(imageRawData)
|
||||
imgSize = int(imgBytes / 1024)
|
||||
imgb64 = base64.b64encode(imageRawData).decode("utf-8")
|
||||
imgDataUri = f"data:image/webp;base64,{imgb64}"
|
||||
|
||||
if ENABLE_IMAGES_CACHING and imgSize >= MIN_IMAGE_CACHE_SIZE:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.call_soon(lambda: asyncio.ensure_future(uploadAndCacheImage(imgSig, imgDataUri)))
|
||||
except RuntimeError:
|
||||
def run_coro():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
new_loop.run_until_complete(uploadAndCacheImage(imgSig, imgDataUri))
|
||||
new_loop.close()
|
||||
threading.Thread(target=run_coro).start()
|
||||
return imgDataUri
|
||||
|
||||
def convertIMG2Tensor(b64img):
|
||||
imgbytes = base64.b64decode(b64img)
|
||||
image = Image.open(io.BytesIO(imgbytes))
|
||||
imageNP = np.array(image).astype(np.float32) / 255.0
|
||||
tensorImage = torch.from_numpy(imageNP).squeeze()
|
||||
return tensorImage
|
||||
|
||||
def convertImageB64List(imageDataObject):
|
||||
images = ()
|
||||
for result in imageDataObject["data"]:
|
||||
generatedImage = result.get(
|
||||
"imageBase64Data",
|
||||
result.get(
|
||||
"maskImageBase64Data", result.get("guideImageBase64Data", False)
|
||||
),
|
||||
)
|
||||
generatedImage = convertIMG2Tensor(generatedImage)
|
||||
images += (generatedImage,)
|
||||
images = torch.stack(images, dim=0)
|
||||
return images
|
||||
from comfy.model_management import InterruptProcessingException
|
||||
from requests.adapters import HTTPAdapter
|
||||
from datetime import datetime, timedelta
|
||||
from server import PromptServer
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
import requests
|
||||
import hashlib
|
||||
import asyncio
|
||||
import random
|
||||
import base64
|
||||
import torch
|
||||
import uuid
|
||||
import time
|
||||
import json
|
||||
import os
|
||||
import io
|
||||
import threading
|
||||
|
||||
load_dotenv()
|
||||
|
||||
BASEFOLDER = Path(__file__).parent.parent.parent
|
||||
IMAGE_CACHE_FILE = BASEFOLDER / "imagesCache.json"
|
||||
|
||||
if not IMAGE_CACHE_FILE.exists():
|
||||
with open(IMAGE_CACHE_FILE, "w") as f:
|
||||
print("[Debugging] Initializing images cache...")
|
||||
json.dump({}, f)
|
||||
|
||||
RUNWARE_REMBG_OUTPUT_FORMATS = {
|
||||
"outputFormat": (
|
||||
["WEBP", "PNG"],
|
||||
{"default": "WEBP", "tooltip": "Choose the output image format."},
|
||||
)
|
||||
}
|
||||
|
||||
MAX_RETRIES = 4
|
||||
RETRY_COOLDOWNS = [1, 2, 5, 10]
|
||||
|
||||
session = requests.Session()
|
||||
adapter = HTTPAdapter(pool_connections=10, pool_maxsize=10)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
|
||||
def generalRequestWrapper(recaller, *args, **kwargs):
|
||||
for attempt in range(MAX_RETRIES + 1):
|
||||
try:
|
||||
return recaller(*args, **kwargs)
|
||||
except (requests.exceptions.ConnectionError, requests.exceptions.SSLError) as e:
|
||||
if attempt == MAX_RETRIES:
|
||||
raise
|
||||
else:
|
||||
cooldown = RETRY_COOLDOWNS[attempt]
|
||||
print(f"[Warning] Error API Request Failed! Retrying in {cooldown} seconds... (Attempt {attempt+1}/{MAX_RETRIES})")
|
||||
time.sleep(cooldown)
|
||||
continue
|
||||
return False
|
||||
|
||||
def getAPIKey():
|
||||
apiKey = os.getenv("RUNWARE_API_KEY")
|
||||
if apiKey and isinstance(apiKey, str) and len(apiKey) > 24:
|
||||
return apiKey
|
||||
return False
|
||||
|
||||
def getTimeout():
|
||||
timeout = os.getenv("RUNWARE_TIMEOUT")
|
||||
if timeout and timeout.isdigit():
|
||||
return int(timeout)
|
||||
else:
|
||||
timeout = 90
|
||||
os.environ["RUNWARE_TIMEOUT"] = str(timeout)
|
||||
return timeout
|
||||
|
||||
def getOutputQuality():
|
||||
output_quality = os.getenv("RUNWARE_OUTPUT_QUALITY")
|
||||
if output_quality and isinstance(output_quality, str) and output_quality.isdigit():
|
||||
return int(output_quality)
|
||||
else:
|
||||
output_quality = 95
|
||||
os.environ["RUNWARE_OUTPUT_QUALITY"] = str(output_quality)
|
||||
return output_quality
|
||||
|
||||
def getOutputFormat():
|
||||
output_format = os.getenv("RUNWARE_OUTPUT_FORMAT")
|
||||
if output_format and isinstance(output_format, str):
|
||||
return output_format
|
||||
else:
|
||||
output_format = "WEBP"
|
||||
os.environ["RUNWARE_OUTPUT_FORMAT"] = output_format
|
||||
return output_format
|
||||
|
||||
def getEnableImagesCaching():
|
||||
enable_images_caching = os.getenv("RUNWARE_ENABLE_IMAGES_CACHING")
|
||||
if enable_images_caching and enable_images_caching.lower() in ["true", "false"]:
|
||||
return enable_images_caching.lower() == "true"
|
||||
else:
|
||||
enable_images_caching = True
|
||||
os.environ["RUNWARE_ENABLE_IMAGES_CACHING"] = str(enable_images_caching)
|
||||
return enable_images_caching
|
||||
|
||||
def getMinImageCacheSize():
|
||||
min_image_cache_size = os.getenv("RUNWARE_MIN_IMAGE_CACHE_SIZE")
|
||||
if min_image_cache_size and min_image_cache_size.isdigit():
|
||||
return int(min_image_cache_size)
|
||||
else:
|
||||
min_image_cache_size = 150
|
||||
os.environ["RUNWARE_MIN_IMAGE_CACHE_SIZE"] = str(min_image_cache_size)
|
||||
return min_image_cache_size
|
||||
|
||||
def getCustomEndpoint():
|
||||
custom_endpoint = os.getenv("RUNWARE_CUSTOM_ENDPOINT")
|
||||
if custom_endpoint and isinstance(custom_endpoint, str):
|
||||
return custom_endpoint
|
||||
return "https://api.runware.ai/v1"
|
||||
|
||||
SESSION_TIMEOUT = getTimeout()
|
||||
RUNWARE_API_KEY = getAPIKey()
|
||||
OUTPUT_FORMAT = getOutputFormat()
|
||||
OUTPUT_QUALITY = getOutputQuality()
|
||||
ENABLE_IMAGES_CACHING = getEnableImagesCaching()
|
||||
MIN_IMAGE_CACHE_SIZE = getMinImageCacheSize()
|
||||
RUNWARE_API_BASE_URL = getCustomEndpoint()
|
||||
|
||||
def setEnvKey(keyName, keyValue):
|
||||
comfyNodeRoot = Path(__file__).parent.parent.parent
|
||||
envFilePath = comfyNodeRoot / ".env"
|
||||
if not envFilePath.exists():
|
||||
envFilePath.touch()
|
||||
with open(envFilePath, "r") as f:
|
||||
lines = f.readlines()
|
||||
key_exists = False
|
||||
new_lines = []
|
||||
for line in lines:
|
||||
if line.startswith(f"{keyName}="):
|
||||
key_exists = True
|
||||
new_lines.append(f"{keyName}={keyValue}\n")
|
||||
else:
|
||||
new_lines.append(line)
|
||||
if not key_exists:
|
||||
new_lines.append(f"{keyName}={keyValue}\n")
|
||||
with open(envFilePath, "w") as f:
|
||||
f.writelines(new_lines)
|
||||
return True
|
||||
|
||||
def setAPIKey(apiKey: str):
|
||||
global RUNWARE_API_KEY
|
||||
envSetRes = setEnvKey("RUNWARE_API_KEY", apiKey)
|
||||
if envSetRes:
|
||||
RUNWARE_API_KEY = apiKey
|
||||
os.environ["RUNWARE_API_KEY"] = apiKey
|
||||
return True
|
||||
|
||||
def setTimeout(timeout: int):
|
||||
envSetRes = setEnvKey("RUNWARE_TIMEOUT", str(timeout))
|
||||
if envSetRes:
|
||||
global SESSION_TIMEOUT
|
||||
SESSION_TIMEOUT = timeout
|
||||
os.environ["RUNWARE_TIMEOUT"] = str(timeout)
|
||||
return True
|
||||
|
||||
def setOutputFormat(format: str):
|
||||
envSetRes = setEnvKey("RUNWARE_OUTPUT_FORMAT", format)
|
||||
if envSetRes:
|
||||
global OUTPUT_FORMAT
|
||||
OUTPUT_FORMAT = format
|
||||
os.environ["RUNWARE_OUTPUT_FORMAT"] = format
|
||||
return True
|
||||
|
||||
def setOutputQuality(quality: int):
|
||||
envSetRes = setEnvKey("RUNWARE_OUTPUT_QUALITY", str(quality))
|
||||
if envSetRes:
|
||||
global OUTPUT_QUALITY
|
||||
OUTPUT_QUALITY = quality
|
||||
os.environ["RUNWARE_OUTPUT_QUALITY"] = str(quality)
|
||||
return True
|
||||
|
||||
def setEnableImagesCaching(enabled: bool):
|
||||
envSetRes = setEnvKey("RUNWARE_ENABLE_IMAGES_CACHING", str(enabled))
|
||||
if envSetRes:
|
||||
global ENABLE_IMAGES_CACHING
|
||||
ENABLE_IMAGES_CACHING = enabled
|
||||
os.environ["RUNWARE_ENABLE_IMAGES_CACHING"] = str(enabled)
|
||||
return True
|
||||
|
||||
def setMinImageCacheSize(size: int):
|
||||
envSetRes = setEnvKey("RUNWARE_MIN_IMAGE_CACHE_SIZE", str(size))
|
||||
if envSetRes:
|
||||
global MIN_IMAGE_CACHE_SIZE
|
||||
MIN_IMAGE_CACHE_SIZE = size
|
||||
os.environ["RUNWARE_MIN_IMAGE_CACHE_SIZE"] = str(size)
|
||||
return True
|
||||
|
||||
def genRandSeed(minSeed=1000, maxSeed=9223372036854776000):
|
||||
return random.randint(minSeed, maxSeed)
|
||||
|
||||
def genRandUUID():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
def checkAPIKey(apiKey):
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"Accept-Encoding": "gzip, deflate, br, zstd",
|
||||
}
|
||||
|
||||
genConfig = [
|
||||
{
|
||||
"taskType": "authentication",
|
||||
"apiKey": apiKey,
|
||||
}
|
||||
]
|
||||
|
||||
try:
|
||||
|
||||
def recaller():
|
||||
return session.post(
|
||||
RUNWARE_API_BASE_URL,
|
||||
headers=headers,
|
||||
json=genConfig,
|
||||
timeout=10,
|
||||
allow_redirects=False,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
genResult = generalRequestWrapper(recaller)
|
||||
genResult = genResult.json()
|
||||
if "errors" in genResult:
|
||||
return str(genResult["errors"][0]["message"])
|
||||
else:
|
||||
return True
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
def sendImageCaption(captionText, nodeID):
|
||||
PromptServer.instance.send_sync(
|
||||
"runwareImageCaption",
|
||||
{
|
||||
"success": True,
|
||||
"captionText": captionText,
|
||||
"nodeID": nodeID,
|
||||
},
|
||||
)
|
||||
|
||||
def inferenecRequest(genConfig):
|
||||
global RUNWARE_API_KEY, RUNWARE_API_BASE_URL, SESSION_TIMEOUT
|
||||
RUNWARE_API_KEY = os.getenv("RUNWARE_API_KEY")
|
||||
SESSION_TIMEOUT = int(os.getenv("RUNWARE_TIMEOUT"))
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNWARE_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept-Encoding": "gzip, deflate, br, zstd",
|
||||
}
|
||||
|
||||
try:
|
||||
def recaller():
|
||||
return session.post(
|
||||
RUNWARE_API_BASE_URL,
|
||||
headers=headers,
|
||||
json=genConfig,
|
||||
timeout=SESSION_TIMEOUT,
|
||||
allow_redirects=False,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
genResult = generalRequestWrapper(recaller)
|
||||
try:
|
||||
genResult = genResult.json()
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"[Debugging] Runware JSON Decode Error: {str(e)}")
|
||||
print(f"[Debugging] Runware Response Status Code: {genResult.status_code}")
|
||||
print(f"[Debugging] Runware Response Headers: {genResult.headers}")
|
||||
print(f"[Debugging] Runware Raw Response Content: {genResult.content}")
|
||||
raise Exception("Error: Invalid JSON response from API!")
|
||||
if "errors" in genResult:
|
||||
raise Exception(genResult["errors"][0]["message"])
|
||||
else:
|
||||
return genResult
|
||||
except requests.exceptions.Timeout:
|
||||
raise Exception(
|
||||
f"Error: Request Timed Out After {SESSION_TIMEOUT} Seconds - Please Try Again!"
|
||||
)
|
||||
except requests.exceptions.RequestException:
|
||||
raise Exception("Error: Runware Request Failed!")
|
||||
except Exception as e:
|
||||
if "invalid api key" in str(e).lower():
|
||||
PromptServer.instance.send_sync(
|
||||
"runwareError",
|
||||
{
|
||||
"success": False,
|
||||
"errorMessage": str(e),
|
||||
"errorCode": 401,
|
||||
},
|
||||
)
|
||||
raise InterruptProcessingException()
|
||||
else:
|
||||
raise Exception(f"Error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def uploadImage(imageDataUri):
|
||||
uploadTaskConfig = [
|
||||
{"taskType": "imageUpload", "taskUUID": genRandUUID(), "image": imageDataUri}
|
||||
]
|
||||
|
||||
try:
|
||||
uploadResult = inferenecRequest(uploadTaskConfig)
|
||||
if (
|
||||
uploadResult
|
||||
and "data" in uploadResult
|
||||
and "imageUUID" in uploadResult["data"][0]
|
||||
):
|
||||
imageUUID = uploadResult["data"][0]["imageUUID"]
|
||||
return imageUUID
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
async def uploadAndCacheImage(imgSig: str, imgDataUri: str):
|
||||
try:
|
||||
imageUUID = await uploadImage(imgDataUri)
|
||||
if imageUUID:
|
||||
await imageStoreSet(imgSig, imageUUID)
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
async def imageStoreSet(imgHash: str, imgUUID: str) -> bool:
|
||||
try:
|
||||
with open(IMAGE_CACHE_FILE, "r") as f:
|
||||
cache = json.load(f)
|
||||
expires = (datetime.now() + timedelta(days=30)).isoformat()
|
||||
cache[imgHash] = {"uuid": imgUUID, "expires": expires}
|
||||
with open(IMAGE_CACHE_FILE, "w") as f:
|
||||
json.dump(cache, f, indent=2)
|
||||
return True
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
def imageStoreGet(imgHash: str) -> str | bool:
|
||||
try:
|
||||
with open(IMAGE_CACHE_FILE, "r") as f:
|
||||
cache = json.load(f)
|
||||
if imgHash not in cache:
|
||||
return False
|
||||
entry = cache[imgHash]
|
||||
expires = datetime.fromisoformat(entry["expires"])
|
||||
if datetime.now() > expires:
|
||||
del cache[imgHash]
|
||||
with open(IMAGE_CACHE_FILE, "w") as f:
|
||||
json.dump(cache, f, indent=2)
|
||||
return False
|
||||
return entry["uuid"]
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
def convertTensor2IMG(tensorImage):
|
||||
global ENABLE_IMAGES_CACHING, MIN_IMAGE_CACHE_SIZE
|
||||
ENABLE_IMAGES_CACHING = getEnableImagesCaching()
|
||||
MIN_IMAGE_CACHE_SIZE = int(os.getenv("RUNWARE_MIN_IMAGE_CACHE_SIZE"))
|
||||
|
||||
imageNP = (tensorImage.squeeze().numpy() * 255).astype(np.uint8)
|
||||
imgSig = hashlib.sha256(imageNP.tobytes()).hexdigest()
|
||||
|
||||
if ENABLE_IMAGES_CACHING:
|
||||
imgUUID = imageStoreGet(imgSig)
|
||||
if imgUUID:
|
||||
return imgUUID
|
||||
|
||||
image = Image.fromarray(imageNP)
|
||||
with io.BytesIO() as buffer:
|
||||
image.save(
|
||||
buffer, format="webp", quality=100, subsampling=0, method=6, exact=True
|
||||
)
|
||||
imageRawData = buffer.getvalue()
|
||||
imgBytes = len(imageRawData)
|
||||
imgSize = int(imgBytes / 1024)
|
||||
imgb64 = base64.b64encode(imageRawData).decode("utf-8")
|
||||
imgDataUri = f"data:image/webp;base64,{imgb64}"
|
||||
|
||||
if ENABLE_IMAGES_CACHING and imgSize >= MIN_IMAGE_CACHE_SIZE:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.call_soon(lambda: asyncio.ensure_future(uploadAndCacheImage(imgSig, imgDataUri)))
|
||||
except RuntimeError:
|
||||
def run_coro():
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
new_loop.run_until_complete(uploadAndCacheImage(imgSig, imgDataUri))
|
||||
new_loop.close()
|
||||
threading.Thread(target=run_coro).start()
|
||||
return imgDataUri
|
||||
|
||||
|
||||
|
||||
def convertTensor2IMGForVideo(tensorImage):
|
||||
"""Convert tensor to image and force upload to get UUID for video frame images"""
|
||||
imageNP = (tensorImage.squeeze().numpy() * 255).astype(np.uint8)
|
||||
imgSig = hashlib.sha256(imageNP.tobytes()).hexdigest()
|
||||
|
||||
# Check if already cached
|
||||
imgUUID = imageStoreGet(imgSig)
|
||||
if imgUUID:
|
||||
return imgUUID
|
||||
|
||||
# Convert to data URI - use PNG for video frame images
|
||||
image = Image.fromarray(imageNP)
|
||||
with io.BytesIO() as buffer:
|
||||
image.save(buffer, format="PNG")
|
||||
imageRawData = buffer.getvalue()
|
||||
imgb64 = base64.b64encode(imageRawData).decode("utf-8")
|
||||
imgDataUri = f"data:image/png;base64,{imgb64}"
|
||||
|
||||
# Force upload to get UUID - use synchronous approach to avoid event loop conflicts
|
||||
try:
|
||||
# Use synchronous upload instead of async
|
||||
uploadTaskConfig = [
|
||||
{"taskType": "imageUpload", "taskUUID": genRandUUID(), "image": imgDataUri}
|
||||
]
|
||||
|
||||
uploadResult = inferenecRequest(uploadTaskConfig)
|
||||
if (
|
||||
uploadResult
|
||||
and "data" in uploadResult
|
||||
and "imageUUID" in uploadResult["data"][0]
|
||||
):
|
||||
uploaded_uuid = uploadResult["data"][0]["imageUUID"]
|
||||
|
||||
# Cache the UUID synchronously
|
||||
try:
|
||||
with open(IMAGE_CACHE_FILE, "r") as f:
|
||||
cache = json.load(f)
|
||||
expires = (datetime.now() + timedelta(days=30)).isoformat()
|
||||
cache[imgSig] = {"uuid": uploaded_uuid, "expires": expires}
|
||||
with open(IMAGE_CACHE_FILE, "w") as f:
|
||||
json.dump(cache, f, indent=2)
|
||||
except Exception as e:
|
||||
print(f"[Warning] Error caching image: {e}")
|
||||
|
||||
return uploaded_uuid
|
||||
else:
|
||||
print("[Warning] Failed to upload image, returning data URI")
|
||||
return imgDataUri
|
||||
except Exception as e:
|
||||
print(f"[Warning] Error uploading image: {e}")
|
||||
return imgDataUri
|
||||
|
||||
def convertIMG2Tensor(b64img):
|
||||
imgbytes = base64.b64decode(b64img)
|
||||
image = Image.open(io.BytesIO(imgbytes))
|
||||
imageNP = np.array(image).astype(np.float32) / 255.0
|
||||
tensorImage = torch.from_numpy(imageNP).squeeze()
|
||||
return tensorImage
|
||||
|
||||
def convertImageB64List(imageDataObject):
|
||||
images = ()
|
||||
for result in imageDataObject["data"]:
|
||||
generatedImage = result.get(
|
||||
"imageBase64Data",
|
||||
result.get(
|
||||
"maskImageBase64Data", result.get("guideImageBase64Data", False)
|
||||
),
|
||||
)
|
||||
generatedImage = convertIMG2Tensor(generatedImage)
|
||||
images += (generatedImage,)
|
||||
images = torch.stack(images, dim=0)
|
||||
return images
|
||||
|
||||
class VideoObject:
|
||||
def __init__(self, video_url, width=None, height=None):
|
||||
self.video_url = video_url
|
||||
# Use provided dimensions or default to 864x480 for Seedance Lite
|
||||
self.width = width if width is not None else 864
|
||||
self.height = height if height is not None else 480
|
||||
|
||||
def get_dimensions(self):
|
||||
return (self.width, self.height)
|
||||
|
||||
def save_to(self, filename, **kwargs):
|
||||
"""Save video to file by downloading from URL with retry logic"""
|
||||
|
||||
|
||||
while True:
|
||||
try:
|
||||
response = requests.get(self.video_url, stream=True, timeout=30)
|
||||
print(f"[Debugging] Response: {response}")
|
||||
|
||||
if response.status_code == 422:
|
||||
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
with open(filename, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
def __str__(self):
|
||||
return f"VideoObject(url={self.video_url}, dimensions={self.width}x{self.height})"
|
||||
|
||||
def convertVideoB64List(videoDataObject, width=None, height=None):
|
||||
videos = ()
|
||||
for result in videoDataObject["data"]:
|
||||
generatedVideo = result.get("videoBase64Data", False)
|
||||
if generatedVideo:
|
||||
# For base64 data, create a video object (would need proper decoding in full implementation)
|
||||
video_obj = VideoObject(f"data:video/mp4;base64,{generatedVideo}", width, height)
|
||||
videos += (video_obj,)
|
||||
else:
|
||||
# If no base64 data, try to get video URL
|
||||
videoURL = result.get("videoURL", False)
|
||||
if videoURL:
|
||||
# Create a proper video object that ComfyUI can handle
|
||||
video_obj = VideoObject(videoURL, width, height)
|
||||
videos += (video_obj,)
|
||||
return videos
|
||||
|
||||
def pollVideoResult(taskUUID):
|
||||
"""Poll for video generation result using task UUID with getResponse"""
|
||||
global RUNWARE_API_KEY, RUNWARE_API_BASE_URL, SESSION_TIMEOUT
|
||||
|
||||
pollConfig = [
|
||||
{
|
||||
"taskType": "getResponse",
|
||||
"taskUUID": taskUUID,
|
||||
}
|
||||
]
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {RUNWARE_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept-Encoding": "gzip, deflate, br, zstd",
|
||||
}
|
||||
|
||||
try:
|
||||
def recaller():
|
||||
return session.post(
|
||||
RUNWARE_API_BASE_URL,
|
||||
headers=headers,
|
||||
json=pollConfig,
|
||||
timeout=30, # Shorter timeout for polling
|
||||
allow_redirects=False,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
pollResult = generalRequestWrapper(recaller)
|
||||
try:
|
||||
pollResult = pollResult.json()
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"[Debugging] Runware Poll JSON Decode Error: {str(e)}")
|
||||
return None
|
||||
|
||||
if "errors" in pollResult:
|
||||
print(f"[Debugging] Poll error: {pollResult['errors'][0]['message']}")
|
||||
return None
|
||||
else:
|
||||
return pollResult
|
||||
except Exception as e:
|
||||
print(f"[Debugging] Poll request failed: {str(e)}")
|
||||
return None
|
||||
|
||||
@@ -10,6 +10,7 @@ class vaeSearch:
|
||||
"All",
|
||||
"FLUX.1-Schnell",
|
||||
"FLUX.1-Dev",
|
||||
"FLUX.1 Krea",
|
||||
"Pony",
|
||||
"SD 1.5",
|
||||
"SD 1.5 Hyper",
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
from .utils import runwareUtils as rwUtils
|
||||
from .videoModelSearch import videoModelSearch
|
||||
|
||||
|
||||
|
||||
class txt2vid:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"Model": ("RUNWAREVIDEOMODEL", {
|
||||
"tooltip": "Connect a Runware Video Model From Runware Video Model Search Node.",
|
||||
}),
|
||||
"positivePrompt": ("STRING", {
|
||||
"multiline": True,
|
||||
"placeholder": "Positive Prompt: a text instruction to guide the model on generating the video. It is usually a sentence or a paragraph that provides positive guidance for the task. This parameter is essential to shape the desired results.\n\nYou Can Press (Ctrl + Alt + E) To Enhance The Prompt!",
|
||||
"tooltip": "Positive Prompt: a text instruction to guide the model on generating the video. You Can Also Press (Ctrl + Alt + E) To Enhance The Prompt!"
|
||||
}),
|
||||
"negativePrompt": ("STRING", {
|
||||
"multiline": True,
|
||||
"placeholder": "Negative Prompt: a text instruction to guide the model on generating the video. It is usually a sentence or a paragraph that provides negative guidance for the task. This parameter helps to avoid certain undesired results.",
|
||||
"tooltip": "Negative Prompt: a text instruction to guide the model on generating the video."
|
||||
}),
|
||||
"Multi Inference Mode": ("BOOLEAN", {
|
||||
"tooltip": "If Enabled the node will skip the video generation process and will only return the Runware Task Object to be used in the Multi Inference Node.",
|
||||
"default": False,
|
||||
"label_on": "Enabled",
|
||||
"label_off": "Disabled",
|
||||
}),
|
||||
"Prompt Weighting": (["Disabled", "sdEmbeds", "Compel"], {
|
||||
"default": "Disabled",
|
||||
"tooltip": "Prompt weighting allows you to adjust how strongly different parts of your prompt influence the generated video.\n\nChoose between \"compel\" notation with advanced weighting operations or \"sdEmbeds\" for simple emphasis adjustments.\n\nCompel Example: \"small+ dog, pixar style\"\n\nsdEmbeds Example: \"(small:2.5) dog, pixar style\"",
|
||||
}),
|
||||
"useCustomDimensions": ("BOOLEAN", {
|
||||
"tooltip": "Model Default: Uses optimal dimensions for selected model. Custom: Uses width/height values below.",
|
||||
"default": False,
|
||||
"label_on": "Custom",
|
||||
"label_off": "Model Default",
|
||||
}),
|
||||
"width": ("INT", {
|
||||
"tooltip": "Width in pixels. Only used when 'Custom' is selected above.",
|
||||
"default": 1024,
|
||||
"min": 256,
|
||||
"max": 1920,
|
||||
"step": 1,
|
||||
}),
|
||||
"height": ("INT", {
|
||||
"tooltip": "Height in pixels. Only used when 'Custom' is selected above.",
|
||||
"default": 576,
|
||||
"min": 256,
|
||||
"max": 1920,
|
||||
"step": 1,
|
||||
}),
|
||||
"duration": ("INT", {
|
||||
"tooltip": "The duration of the video in seconds.",
|
||||
"default": 5,
|
||||
"min": 1,
|
||||
"max": 30,
|
||||
}),
|
||||
"fps": ("INT", {
|
||||
"tooltip": "Frames per second for the generated video.",
|
||||
"default": 24,
|
||||
"min": 8,
|
||||
"max": 60,
|
||||
}),
|
||||
"outputFormat": (["mp4", "webm", "mov"], {
|
||||
"default": "mp4",
|
||||
"tooltip": "Choose the output video format.",
|
||||
}),
|
||||
"batchSize": ("INT", {
|
||||
"tooltip": "The number of videos to generate in a single request.",
|
||||
"default": 1,
|
||||
"min": 1,
|
||||
"max": 4,
|
||||
}),
|
||||
"includeCost": ("BOOLEAN", {
|
||||
"tooltip": "Include cost information in the response.",
|
||||
"default": True,
|
||||
"label_on": "Enabled",
|
||||
"label_off": "Disabled",
|
||||
}),
|
||||
"seed": ("INT", {
|
||||
"tooltip": "A value used to randomize the video generation. If you want to make videos reproducible (generate the same video multiple times), you can use the same seed value. Note: Only supported by Wan models.",
|
||||
"default": rwUtils.genRandSeed(),
|
||||
"min": 1,
|
||||
"max": 9223372036854776000,
|
||||
}),
|
||||
},
|
||||
"optional": {
|
||||
"firstImage": ("IMAGE", {
|
||||
"tooltip": "Image to use as the first frame of the video.",
|
||||
}),
|
||||
"lastImage": ("IMAGE", {
|
||||
"tooltip": "Image to use as the last frame of the video.",
|
||||
}),
|
||||
"providerSettings": ("DICT", {
|
||||
"tooltip": "Provider-specific settings for video generation.",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, positivePrompt, negativePrompt):
|
||||
if (positivePrompt is not None and (positivePrompt == "" or len(positivePrompt) < 3 or len(positivePrompt) > 2000)):
|
||||
raise Exception(
|
||||
"Positive Prompt Must Be Between 3 And 2000 characters!")
|
||||
if (negativePrompt is not None and negativePrompt != "" and (len(negativePrompt) < 3 or len(negativePrompt) > 2000)):
|
||||
raise Exception(
|
||||
"Negative Prompt Must Be Between 3 And 2000 characters!")
|
||||
return True
|
||||
|
||||
DESCRIPTION = "Generates Videos Lightning Fast With Runware Video Inference Engine."
|
||||
FUNCTION = "generateVideo"
|
||||
RETURN_TYPES = ("VIDEO", "RUNWARETASK")
|
||||
RETURN_NAMES = ("VIDEO", "RW-Task")
|
||||
CATEGORY = "Runware"
|
||||
|
||||
def generateVideo(self, **kwargs):
|
||||
runwareVideoModel = kwargs.get("Model")
|
||||
positivePrompt = kwargs.get("positivePrompt")
|
||||
negativePrompt = kwargs.get("negativePrompt", None)
|
||||
multiInferenceMode = kwargs.get("Multi Inference Mode", False)
|
||||
promptWeighting = kwargs.get("Prompt Weighting", "Disabled")
|
||||
providerSettings = kwargs.get("providerSettings", None)
|
||||
firstImage = kwargs.get("firstImage", None)
|
||||
lastImage = kwargs.get("lastImage", None)
|
||||
useCustomDimensions = kwargs.get("useCustomDimensions", False)
|
||||
customWidth = kwargs.get("width", 864)
|
||||
customHeight = kwargs.get("height", 480)
|
||||
duration = kwargs.get("duration", 5)
|
||||
fps = kwargs.get("fps", 24)
|
||||
outputFormat = kwargs.get("outputFormat", "mp4")
|
||||
batchSize = kwargs.get("batchSize", 1)
|
||||
includeCost = kwargs.get("includeCost", True)
|
||||
seed = kwargs.get("seed", 1)
|
||||
|
||||
# Handle model input - could be dict or string
|
||||
if isinstance(runwareVideoModel, dict):
|
||||
model = runwareVideoModel.get("model", "")
|
||||
else:
|
||||
model = runwareVideoModel
|
||||
|
||||
# Determine dimensions based on custom setting
|
||||
if useCustomDimensions:
|
||||
width = customWidth
|
||||
height = customHeight
|
||||
else:
|
||||
|
||||
|
||||
model_dimensions = videoModelSearch.MODEL_DIMENSIONS.get(model, {"width": 1024, "height": 576})
|
||||
width = model_dimensions["width"]
|
||||
height = model_dimensions["height"]
|
||||
|
||||
genConfig = [
|
||||
{
|
||||
"taskType": "videoInference",
|
||||
"taskUUID": rwUtils.genRandUUID(),
|
||||
"positivePrompt": positivePrompt,
|
||||
"height": height,
|
||||
"width": width,
|
||||
"model": model,
|
||||
"fps": fps,
|
||||
"outputFormat": outputFormat,
|
||||
"numberResults": batchSize,
|
||||
"includeCost": includeCost,
|
||||
}
|
||||
]
|
||||
|
||||
# Add duration parameter - unified for all models
|
||||
genConfig[0]["duration"] = duration
|
||||
|
||||
# Add seed parameter only for supported models
|
||||
|
||||
if model in videoModelSearch.SEED_SUPPORTED_MODELS:
|
||||
genConfig[0]["seed"] = seed
|
||||
|
||||
if (negativePrompt is not None and negativePrompt != ""):
|
||||
genConfig[0]["negativePrompt"] = negativePrompt
|
||||
if (promptWeighting != "Disabled"):
|
||||
if (promptWeighting == "sdEmbeds"):
|
||||
genConfig[0]["promptWeighting"] = "sdEmbeds"
|
||||
else:
|
||||
genConfig[0]["promptWeighting"] = "compel"
|
||||
# Build frameImages array from firstImage and lastImage parameters
|
||||
frameImages = []
|
||||
if (firstImage is not None):
|
||||
# Force upload to get UUID for video frame images
|
||||
firstImageUUID = rwUtils.convertTensor2IMGForVideo(firstImage)
|
||||
# Construct full URL with PNG extension
|
||||
firstImageURL = f"https://im.runware.ai/image/ii/{firstImageUUID}.webp"
|
||||
print(f"[Debugging] First image URL: {firstImageURL}")
|
||||
frameImages.append({
|
||||
"inputImage": firstImageURL
|
||||
})
|
||||
|
||||
if (lastImage is not None):
|
||||
# Force upload to get UUID for video frame images
|
||||
lastImageUUID = rwUtils.convertTensor2IMGForVideo(lastImage)
|
||||
# Construct full URL with PNG extension
|
||||
lastImageURL = f"https://im.runware.ai/image/ii/{lastImageUUID}.webp"
|
||||
print(f"[Debugging] Last image URL: {lastImageURL}")
|
||||
frameImages.append({
|
||||
"inputImage": lastImageURL
|
||||
})
|
||||
|
||||
if frameImages:
|
||||
genConfig[0]["frameImages"] = frameImages
|
||||
print(f"[Debugging] Frame images array: {frameImages}")
|
||||
if (providerSettings is not None):
|
||||
genConfig[0]["providerSettings"] = providerSettings
|
||||
|
||||
if (multiInferenceMode):
|
||||
return (None, genConfig)
|
||||
else:
|
||||
try:
|
||||
genResult = rwUtils.inferenecRequest(genConfig)
|
||||
print(f"[Debugging] Generation config: {genConfig}")
|
||||
except Exception as e:
|
||||
# Check if it's a dimension error and provide helpful information
|
||||
error_msg = str(e)
|
||||
if ("unsupported width" in error_msg.lower() or
|
||||
"unsupported height" in error_msg.lower() or
|
||||
"invalid width" in error_msg.lower() or
|
||||
"invalid height" in error_msg.lower() or
|
||||
"dimension not supported" in error_msg.lower()):
|
||||
# Get model default dimensions for comparison
|
||||
model_dimensions = videoModelSearch.MODEL_DIMENSIONS.get(model, {"width": 1024, "height": 576})
|
||||
expected_width = model_dimensions["width"]
|
||||
expected_height = model_dimensions["height"]
|
||||
|
||||
raise Exception(f"Error: Unsupported width/height combination for this model architecture. You used {width}x{height}, but {model} expects {expected_width}x{expected_height}. Please use 'Model Default' or set custom dimensions to {expected_width}x{expected_height}.")
|
||||
else:
|
||||
# Re-raise the original error
|
||||
raise e
|
||||
|
||||
# Extract task UUID for polling
|
||||
taskUUID = genConfig[0]["taskUUID"]
|
||||
|
||||
# Poll for video completion
|
||||
while True:
|
||||
|
||||
# Poll for video result
|
||||
pollResult = rwUtils.pollVideoResult(taskUUID)
|
||||
print(f"[Debugging] Poll result: {pollResult}")
|
||||
|
||||
if pollResult and "data" in pollResult and len(pollResult["data"]) > 0:
|
||||
video_data = pollResult["data"][0]
|
||||
|
||||
# Check status directly
|
||||
if "status" in video_data:
|
||||
status = video_data["status"]
|
||||
|
||||
if status == "success":
|
||||
if "videoURL" in video_data or "videoBase64Data" in video_data:
|
||||
videos = rwUtils.convertVideoB64List(pollResult, width, height)
|
||||
return videos
|
||||
elif status != "processing":
|
||||
raise Exception(f"Video generation failed: {video_data.get('error', 'Unknown error')}")
|
||||
# If status is "processing", continue polling
|
||||
|
||||
# Wait before next poll
|
||||
rwUtils.time.sleep(1)
|
||||
@@ -0,0 +1,163 @@
|
||||
class videoModelSearch:
|
||||
# Define the video models by architecture with their default dimensions
|
||||
VIDEO_MODELS = {
|
||||
"KlingAI": [
|
||||
"urn:air:video:klingai:1@2 (KlingAI V1.0 Pro)",
|
||||
"urn:air:video:klingai:1@1 (KlingAI V1 Standard)",
|
||||
"urn:air:video:klingai:2@2 (KlingAI V1.5 Pro)",
|
||||
"urn:air:video:klingai:2@1 (KlingAI V1.5 Standard)",
|
||||
"urn:air:video:klingai:3@1 (KlingAI V1.6 Standard)",
|
||||
"urn:air:video:klingai:3@2 (KlingAI V1.6 Pro)",
|
||||
"urn:air:video:klingai:4@3 (KlingAI V2.1 Master)",
|
||||
"urn:air:video:klingai:5@1 (KlingAI V2.1 Standard (I2V))",
|
||||
"urn:air:video:klingai:5@2 (KlingAI V2.1 Pro (I2V))",
|
||||
"urn:air:video:klingai:5@3 (KlingAI V2.0 Master)",
|
||||
],
|
||||
"Veo": [
|
||||
"urn:air:video:google:2@0 (Veo 2.0)",
|
||||
"urn:air:video:google:3@0 (Veo 3.0)",
|
||||
"urn:air:video:google:3@1 (Veo 3.0 Fast)",
|
||||
],
|
||||
"Seedance": [
|
||||
"urn:air:video:bytedance:2@1 (Seedance 1.0 Pro)",
|
||||
"urn:air:video:bytedance:1@1 (Seedance 1.0 Lite)",
|
||||
],
|
||||
"MiniMax": [
|
||||
"urn:air:video:minimax:1@1 (MiniMax 01 Base)",
|
||||
"urn:air:video:minimax:2@1 (MiniMax 01 Director)",
|
||||
"urn:air:video:minimax:2@3 (MiniMax I2V 01 Live)",
|
||||
"urn:air:video:minimax:3@1 (MiniMax 02 Hailuo)",
|
||||
],
|
||||
"PixVerse": [
|
||||
"urn:air:video:pixverse:1@1 (PixVerse v3.5)",
|
||||
"urn:air:video:pixverse:1@2 (PixVerse v4)",
|
||||
"urn:air:video:pixverse:1@3 (PixVerse v4.5)",
|
||||
],
|
||||
"Vidu": [
|
||||
"urn:air:video:vidu:1@0 (Vidu Q1 Classic)",
|
||||
"urn:air:video:vidu:1@1 (Vidu Q1)",
|
||||
"urn:air:video:vidu:1@5 (Vidu 1.5)",
|
||||
"urn:air:video:vidu:2@0 (Vidu 2.0)",
|
||||
],
|
||||
"Wan": [
|
||||
"urn:air:video:runware:200@1 (Wan 2.1 1.3B)",
|
||||
"urn:air:video:runware:200@2 (Wan 2.1 14B)",
|
||||
],
|
||||
}
|
||||
|
||||
# Model dimensions mapping
|
||||
MODEL_DIMENSIONS = {
|
||||
# KlingAI Models
|
||||
"urn:air:video:klingai:1@2": {"width": 1280, "height": 720}, # KlingAI V1.0 Pro
|
||||
"urn:air:video:klingai:1@1": {"width": 1280, "height": 720}, # KlingAI V1 Standard
|
||||
"urn:air:video:klingai:2@2": {"width": 1920, "height": 1080}, # KlingAI V1.5 Pro
|
||||
"urn:air:video:klingai:2@1": {"width": 1280, "height": 720}, # KlingAI V1.5 Standard
|
||||
"urn:air:video:klingai:3@1": {"width": 1280, "height": 720}, # KlingAI V1.6 Standard
|
||||
"urn:air:video:klingai:3@2": {"width": 1920, "height": 1080}, # KlingAI V1.6 Pro
|
||||
"urn:air:video:klingai:4@3": {"width": 1280, "height": 720}, # KlingAI V2.1 Master
|
||||
"urn:air:video:klingai:5@1": {"width": 1280, "height": 720}, # KlingAI V2.1 Standard (I2V)
|
||||
"urn:air:video:klingai:5@2": {"width": 1920, "height": 1080}, # KlingAI V2.1 Pro (I2V)
|
||||
"urn:air:video:klingai:5@3": {"width": 1920, "height": 1080}, # KlingAI V2.0 Master
|
||||
|
||||
# Veo Models
|
||||
"urn:air:video:google:2@0": {"width": 1280, "height": 720}, # Veo 2.0
|
||||
"urn:air:video:google:3@0": {"width": 1280, "height": 720}, # Veo 3.0
|
||||
"urn:air:video:google:3@1": {"width": 1280, "height": 720}, # Veo 3.0 Fast
|
||||
|
||||
# Seedance Models
|
||||
"urn:air:video:bytedance:2@1": {"width": 864, "height": 480}, # Seedance 1.0 Pro
|
||||
"urn:air:video:bytedance:1@1": {"width": 864, "height": 480}, # Seedance 1.0 Lite
|
||||
|
||||
# MiniMax Models
|
||||
"urn:air:video:minimax:1@1": {"width": 1366, "height": 768}, # MiniMax 01 Base
|
||||
"urn:air:video:minimax:2@1": {"width": 1366, "height": 768}, # MiniMax 01 Director
|
||||
"urn:air:video:minimax:2@3": {"width": 1366, "height": 768}, # MiniMax I2V 01 Live
|
||||
"urn:air:video:minimax:3@1": {"width": 1366, "height": 768}, # MiniMax 02 Hailuo
|
||||
|
||||
# PixVerse Models
|
||||
"urn:air:video:pixverse:1@1": {"width": 640, "height": 360}, # PixVerse v3.5
|
||||
"urn:air:video:pixverse:1@2": {"width": 640, "height": 360}, # PixVerse v4
|
||||
"urn:air:video:pixverse:1@3": {"width": 640, "height": 360}, # PixVerse v4.5
|
||||
|
||||
# Vidu Models
|
||||
"urn:air:video:vidu:1@0": {"width": 1920, "height": 1080}, # Vidu Q1 Classic
|
||||
"urn:air:video:vidu:1@1": {"width": 1920, "height": 1080}, # Vidu Q1
|
||||
"urn:air:video:vidu:1@5": {"width": 1920, "height": 1080}, # Vidu 1.5
|
||||
"urn:air:video:vidu:2@0": {"width": 1920, "height": 1080}, # Vidu 2.0
|
||||
|
||||
# Wan Models
|
||||
"urn:air:video:runware:200@1": {"width": 853, "height": 480}, # Wan 2.1 1.3B
|
||||
"urn:air:video:runware:200@2": {"width": 853, "height": 480}, # Wan 2.1 14B
|
||||
}
|
||||
|
||||
# Models that support seed parameter
|
||||
SEED_SUPPORTED_MODELS = [
|
||||
"urn:air:video:runware:200@1", # Wan 2.1 1.3B
|
||||
"urn:air:video:runware:200@2", # Wan 2.1 14B
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
# Get all models for "All" option
|
||||
all_models = []
|
||||
for models in cls.VIDEO_MODELS.values():
|
||||
all_models.extend(models)
|
||||
|
||||
return {
|
||||
"required": {
|
||||
"Model Search": ("STRING", {
|
||||
"tooltip": "Search For Specific Video Model By Name Or Civit AIR Code (eg: KlingAI, Veo).",
|
||||
}),
|
||||
"Model Architecture": ([
|
||||
"All",
|
||||
"KlingAI",
|
||||
"Veo",
|
||||
"Seedance",
|
||||
"MiniMax",
|
||||
"PixVerse",
|
||||
"Vidu",
|
||||
"Wan"
|
||||
], {
|
||||
"tooltip": "Choose Video Model Architecture To Filter Results.",
|
||||
"default": "KlingAI",
|
||||
}),
|
||||
"VideoList": (all_models, {
|
||||
"tooltip": "Video Model Results Will Show UP Here So You Could Choose From.",
|
||||
"default": "urn:air:video:klingai:5@3 (KlingAI 2.1)",
|
||||
}),
|
||||
"Use Search Value": ("BOOLEAN", {
|
||||
"tooltip": "When Enabled, the value you've set in the search input will be used instead.\n\nThis is useful in case the model search API is down or you prefer to set the model manually.",
|
||||
"default": False,
|
||||
"label_on": "Enabled",
|
||||
"label_off": "Disabled",
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
DESCRIPTION = "Directly Search and Connect Video Models to Runware Video Inference Nodes In ComfyUI."
|
||||
FUNCTION = "videoModelSearch"
|
||||
RETURN_TYPES = ("RUNWAREVIDEOMODEL", "INT", "INT")
|
||||
RETURN_NAMES = ("Runware Video Model", "Width", "Height")
|
||||
CATEGORY = "Runware"
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, VideoList):
|
||||
return True
|
||||
|
||||
def videoModelSearch(self, **kwargs):
|
||||
enableSearchValue = kwargs.get("Use Search Value", False)
|
||||
searchInput = kwargs.get("Model Search")
|
||||
|
||||
if enableSearchValue:
|
||||
modelAIRCode = searchInput
|
||||
else:
|
||||
crModel = kwargs.get("VideoList")
|
||||
# Extract the full AIR identifier (everything before the first space)
|
||||
modelAIRCode = crModel.split(" (")[0]
|
||||
|
||||
# Get dimensions for the selected model
|
||||
dimensions = self.MODEL_DIMENSIONS.get(modelAIRCode, {"width": 1024, "height": 576})
|
||||
|
||||
return ({
|
||||
"model": modelAIRCode,
|
||||
}, dimensions["width"], dimensions["height"])
|
||||
Reference in New Issue
Block a user