nodes/KwaiVGI-ComfyUI-KLingAI-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[feat] support video extend and lip sync.

Browse Source
This commit is contained in:
chentiejun
2025-01-03 12:19:14 +08:00
parent 9d86ef8342
commit a65eb090f8
10 changed files with 275 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+12
View File
@@ -57,6 +57,18 @@ This node is used to display the try-on effect.
<img src="./examples/kolors_virtual_try_on.png" alt="text2video">
</p>
### Video Extend
This node is used to extend a video.
<p align="center">
<img src="./examples/video_extend.png" alt="video_extend">
</p>
### Lip Sync
This node is used to generate a lip sync video.
<p align="center">
<img src="./examples/lip_sync.png" alt="lip_sync">
</p>
## Pricing
For pricing, follow [KLing AI Pricing](https://klingai.com/dev-center).
examples/image2video.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 168 KiB

After

Width:  |  Height:  |  Size: 483 KiB

examples/lip_sync.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 526 KiB

examples/video_extend.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 753 KiB

py/__init__.py
+6 -2
View File
@@ -1,4 +1,4 @@
from .nodes import Text2VideoNode, ImageGeneratorNode, Image2VideoNode, KLingAIAPIClient, PreviewVideo, KolorsVirtualTryOnNode
from .nodes import Text2VideoNode, ImageGeneratorNode, Image2VideoNode, KLingAIAPIClient, PreviewVideo, KolorsVirtualTryOnNode, VideoExtendNode, LipSyncNode, LipSyncTextInputNode, LipSyncAudioInputNode
NODE_CLASS_MAPPINGS = {
'Client': KLingAIAPIClient,
@@ -6,5 +6,9 @@ NODE_CLASS_MAPPINGS = {
'Text2Video': Text2VideoNode,
'Image2Video': Image2VideoNode,
'Virtual Try On': KolorsVirtualTryOnNode,
'KLingAI Preview Video': PreviewVideo
'KLingAI Preview Video': PreviewVideo,
'Video Extend': VideoExtendNode,
'Lip Sync': LipSyncNode,
'Lip Sync Text Input': LipSyncTextInputNode,
'Lip Sync Audio Input': LipSyncAudioInputNode,
}
py/api/__init__.py
+2
View File
@@ -4,3 +4,5 @@ from .image_generator import ImageGenerator
from .text2video import Text2Video, CameraControl, CameraControlConfig
from .image2video import Image2Video
from .virtual_try_on import KolorsVurtualTryOn
from .video_entend import VideoExtend
from .lip_sync import LipSync, LipSyncInput
py/api/lip_sync.py
+25
View File
@@ -0,0 +1,25 @@
from .prediction import Prediction, VideoPredictionResponse
class LipSyncInput(Prediction):
video_id: str
mode: str
text: str
voice_id: str
voice_language: str
voice_speed: float
audio_type: str
audio_file: str
audio_url: str
class LipSync(Prediction):
input: LipSyncInput
def __init__(self):
super().__init__()
self._request_method = "POST"
self._request_path = "/v1/videos/lip-sync"
self._query_prediction_info_method = "GET"
self._query_prediction_info_path = "/v1/videos/lip-sync"
self._response_cls = VideoPredictionResponse
py/api/prediction.py
-1
View File
@@ -95,7 +95,6 @@ class Prediction:
path = self._query_prediction_info_path + "/" + task_id
resp = client.request(method=self._query_prediction_info_method, path=path)
self._task_info = self._response_cls(**resp.get("data"))
print(f'query: {path}, status: {self._task_info.task_status}')
def run(self, client: Client):
resp = client.request(method=self._request_method, path=self._request_path, json=self.to_dict())
py/api/video_entend.py
+15
View File
@@ -0,0 +1,15 @@
from .prediction import Prediction, VideoPredictionResponse
class VideoExtend(Prediction):
video_id: str
prompt: str
def __init__(self):
super().__init__()
self._request_method = "POST"
self._request_path = "/v1/videos/video-extend"
self._query_prediction_info_method = "GET"
self._query_prediction_info_path = "/v1/videos/video-extend"
self._response_cls = VideoPredictionResponse
py/nodes.py
+215 -15
View File
@@ -1,4 +1,4 @@
from .api import Client, ImageGenerator, Image2Video, Text2Video, CameraControl, CameraControlConfig, KolorsVurtualTryOn
from .api import Client, ImageGenerator, Image2Video, Text2Video, CameraControl, CameraControlConfig, KolorsVurtualTryOn, VideoExtend, LipSync, LipSyncInput
import base64
import io
import os
@@ -203,8 +203,8 @@ class Image2VideoNode:
}
}
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("url",)
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("url", "video_id")
FUNCTION = "generate"
@@ -233,11 +233,11 @@ class Image2VideoNode:
generator.duration = duration
response = generator.run(client)
result = []
for video_info in response.task_result.videos:
result.append(video_info.url)
print(f'KLing API output: {result}')
return (result,)
print(f'KLing API output video id: {video_info.id}, url: {video_info.url}')
return (video_info.url, video_info.id)
return ('', '')
class Text2VideoNode:
@@ -278,8 +278,8 @@ class Text2VideoNode:
}
}
RETURN_TYPES = ("STRING",)
RETURN_NAMES = ("url",)
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("url", "video_id")
FUNCTION = "generate"
@@ -303,7 +303,6 @@ class Text2VideoNode:
generator = Text2Video()
generator.model = model
generator.prompt = prompt
generator.prompt = prompt
generator.negative_prompt = negative_prompt
generator.cfg_scale = cfg_scale
generator.mode = mode
@@ -330,11 +329,11 @@ class Text2VideoNode:
response = generator.run(client)
result = []
for video_info in response.task_result.videos:
result.append(video_info.url)
print(f'KLing API output: {result}')
return (result,)
print(f'KLing API output video id: {video_info.id}, url: {video_info.url}')
return (video_info.url, video_info.id)
return ('', '')
class KolorsVirtualTryOnNode:
@@ -343,7 +342,7 @@ class KolorsVirtualTryOnNode:
return {
"required": {
"client": ("KLING_AI_API_CLIENT",),
"model_name": (["kolors-virtual-try-on-v1"],),
"model_name": (["kolors-virtual-try-on-v1", "kolors-virtual-try-on-v1-5"],),
"human_image": ("IMAGE",),
"cloth_image": ("IMAGE",),
},
@@ -426,3 +425,204 @@ class PreviewVideo:
open(file_path, "wb").write(_fetch_image(video_url))
return {"ui": {"video_url": [video_url]}, "result": (file_path, )}
class VideoExtendNode:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"client": ("KLING_AI_API_CLIENT",),
"video_id": ("STRING", {"multiline": False, "default": ""}),
"prompt": ("STRING", {"multiline": True, "default": ""}),
}
}
OUTPUT_NODE = True
FUNCTION = "run"
CATEGORY = "KLingAI"
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("url", "video_id")
def run(self, client, video_id, prompt):
generator = VideoExtend()
generator.video_id = video_id
generator.prompt = prompt
response = generator.run(client)
for video_info in response.task_result.videos:
print(f'KLing API output video id: {video_info.id}, url: {video_info.url}')
return (video_info.url, video_info.id)
return ('', '')
class LipSyncTextInputNode:
audio_types = {
"阳光少年": "genshin_vindi2",
"懂事小弟": "zhinen_xuesheng",
"运动少年": "tiyuxi_xuedi",
"青春少女": "ai_shatang",
"温柔小妹": "genshin_klee2",
"元气少女": "genshin_kirara",
"阳光男生": "ai_kaiya",
"幽默小哥": "tiexin_nanyou",
"文艺小哥": "ai_chenjiahao_712",
"甜美邻家": "girlfriend_1_speech02",
"温柔姐姐": "chat1_female_new-3",
"职场女青": "girlfriend_2_speech02",
"活泼男童": "cartoon-boy-07",
"俏皮女童": "cartoon-girl-01",
"稳重老爸": "ai_huangyaoshi_712",
"温柔妈妈": "you_pingjing",
"严肃上司": "ai_laoguowang_712",
"优雅贵妇": "chengshu_jiejie",
"慈祥爷爷": "zhuxi_speech02",
"唠叨爷爷": "uk_oldman3",
"唠叨奶奶": "laopopo_speech02",
"和蔼奶奶": "heainainai_speech02",
"东北老铁": "dongbeilaotie_speech02",
"重庆小伙": "chongqingxiaohuo_speech02",
"四川妹子": "chuanmeizi_speech02",
"潮汕大叔": "chaoshandashu_speech02",
"台湾男生": "ai_taiwan_man2_speech02",
"西安掌柜": "xianzhanggui_speech02",
"天津姐姐": "tianjinjiejie_speech02",
"Sunny": "genshin_vindi2",
"Sage": "zhinen_xuesheng",
"Ace": "AOT",
"Blossom": "ai_shatang",
"Peppy": "genshin_klee2",
"Dove": "genshin_kirara",
"Shine": "ai_kaiya",
"Anchor": "oversea_male1",
"Lyric": "ai_chenjiahao_712",
"Melody": "girlfriend_4_speech02",
"Tender": "chat1_female_new-3",
"Siren": "chat_0407_5-1",
"Zippy": "cartoon-boy-07",
"Bud": "uk_boy1",
"Sprite": "cartoon-girl-01",
"Candy": "PeppaPig_platform",
"Beacon": "ai_huangzhong_712",
"Rock": "ai_huangyaoshi_712",
"Titan": "ai_laoguowang_712",
"Grace": "chengshu_jiejie",
"Helen": "you_pingjing",
"Lore": "calm_story1",
"Crag": "uk_man2",
"Prattle": "laopopo_speech02",
"Hearth": "heainainai_speech02"
}
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"text": ("STRING", {"multiline": True, "default": ""}),
"voice_id": (list(LipSyncTextInputNode.audio_types.keys()), {"multiline": False, "default": ""}),
"voice_language": (["zh", "en"], {"multiline": True, "default": "zh"}),
"voice_speed": ("FLOAT", {
"default": 1.0,
"min": 0.8,
"max": 2.0,
"step": 0.1,
"round": 0.01,
"display": "number",
"lazy": True
})
}
}
OUTPUT_NODE = True
FUNCTION = "run"
CATEGORY = "KLingAI"
RETURN_TYPES = ("KLING_AI_API_LIPSYNC_INPUT", )
RETURN_NAMES = ("input", )
def run(self, text, voice_id, voice_language, voice_speed):
input = LipSyncInput()
input.mode = "text2video"
input.text = text
if voice_id in LipSyncTextInputNode.audio_types:
input.voice_id = LipSyncTextInputNode.audio_types[voice_id]
else:
input.voice_id = voice_id
input.voice_language = voice_language
input.voice_speed = voice_speed
return (input, )
class LipSyncAudioInputNode:
@classmethod
def INPUT_TYPES(cls):
return {
"optional": {
"audio_file": ("STRING", {"multiline": False, "default": ""}),
"audio_url": ("STRING", {"multiline": False, "default": ""}),
},
}
OUTPUT_NODE = True
FUNCTION = "run"
CATEGORY = "KLingAI"
RETURN_TYPES = ("KLING_AI_API_LIPSYNC_INPUT", )
RETURN_NAMES = ("input", )
def run(self, audio_file, audio_url):
input = LipSyncInput()
input.mode = "audio2video"
if audio_file is not None and len(audio_file) > 0:
input.audio_type = "file"
if os.path.exists(audio_file):
with open(audio_file, 'rb') as file:
file_data = file.read()
input.audio_file = base64.b64encode(file_data).decode('utf-8')
else:
raise Exception(f"Audio file not found: {audio_file}")
if audio_url is not None and len(audio_url) > 0:
input.audio_type = "url"
input.audio_url = audio_url
return (input, )
class LipSyncNode:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"client": ("KLING_AI_API_CLIENT", ),
"video_id": ("STRING", {"multiline": False, "default": ""}),
"input": ("KLING_AI_API_LIPSYNC_INPUT", )
},
"optional": {
}
}
OUTPUT_NODE = True
FUNCTION = "run"
CATEGORY = "KLingAI"
RETURN_TYPES = ("STRING", "STRING")
RETURN_NAMES = ("url", "video_id")
def run(self, client, video_id, input):
generator = LipSync()
input.video_id = video_id
generator.input = input
response = generator.run(client)
for video_info in response.task_result.videos:
print(f'KLing API output video id: {video_info.id}, url: {video_info.url}')
return (video_info.url, video_info.id)
return ('', '')