feat: Init DashscopeEmoCaller

node.py

@@ -1,9 +1,14 @@
 import os
 import random
 import string
+import time
 from dashscope import Generation, MultiModalConversation
 from http import HTTPStatus
+import requests
 import torchvision.transforms as transforms
+import torchvision.io as io
+import torch
+import numpy as np
 
 
 def generate_random_image_name():
@@ -11,6 +16,11 @@ def generate_random_image_name():
     return f"image_{random_number}.png"
 
 
+def generate_random_video_name():
+    random_number = "".join(random.choices(string.ascii_letters + string.digits, k=10))
+    return f"video_{random_number}.mp4"
+
+
 def get_image_url(image):
     image = image.squeeze(0)
     image = image.permute(2, 0, 1)
@@ -21,6 +31,13 @@ def get_image_url(image):
     return image_name
 
 
+def get_audio_url(audio):
+    random_number = "".join(random.choices(string.ascii_letters + string.digits, k=10))
+    audio_name = f"audio_{random_number}.wav"
+    audio.save(audio_name)
+    return audio_name
+
+
 def get_model_versions(model_type: str) -> list[str]:
     model_versions = []
     current_group = None
@@ -44,6 +61,65 @@ def get_model_versions(model_type: str) -> list[str]:
     return model_versions
 
 
+VIDEO_EXTENSIONS = ["webm", "mp4", "mkv", "gif", "mov"]
+MAX_VIDEO_SIZE_MB = 100
+
+
+def video_to_images(video_path: str, fps: float = None) -> tuple:
+    """将视频转换为图像序列，支持多种格式和内存优化"""
+
+    if not os.path.exists(video_path):
+        raise RuntimeError(f"视频文件不存在: {video_path}")
+
+    if not any(video_path.lower().endswith(ext) for ext in VIDEO_EXTENSIONS):
+        raise RuntimeError(f"不支持的视频格式，支持: {', '.join(VIDEO_EXTENSIONS)}")
+
+    if os.path.getsize(video_path) > MAX_VIDEO_SIZE_MB * 1024 * 1024:
+        raise RuntimeError(f"视频文件过大，最大支持 {MAX_VIDEO_SIZE_MB}MB")
+
+    try:
+        # 读取视频
+        frames, audio_frames, metadata = io.read_video(video_path, pts_unit="sec")
+
+        if frames.shape[0] == 0:
+            raise RuntimeError("视频帧为空")
+
+        # 获取信息
+        frame_count = frames.shape[0]
+        height = frames.shape[1]
+        width = frames.shape[2]
+        source_fps = metadata.get("video", {}).get("fps", 30.0)
+        duration = frame_count / source_fps
+
+        # 调整帧率(如果指定)
+        if fps and fps > 0 and fps != source_fps:
+            target_frames = int(duration * fps)
+            indices = torch.linspace(0, frame_count - 1, target_frames).long()
+            frames = frames[indices]
+            frame_count = len(frames)
+
+        # 格式转换
+        frames = frames.float() / 255.0
+        if frames.shape[-1] == 4:
+            frames = frames[..., :3]
+        frames = frames.permute(0, 3, 1, 2)
+
+        # 返回信息
+        video_info = {
+            "source_fps": source_fps,
+            "source_frame_count": frame_count,
+            "source_duration": duration,
+            "source_width": width,
+            "source_height": height,
+            "target_fps": fps if fps else source_fps,
+        }
+
+        return (frames, frame_count, video_info)
+
+    except Exception as e:
+        raise RuntimeError(f"视频转换失败: {str(e)}")
+
+
 class DashscopeLLMLoader:
     def __init__(self):
         pass
@@ -263,3 +339,188 @@ class DashscopeModelCaller:
 
         message_content = response.output.choices[0].message.content[0]["text"]
         return (message_content,)
+
+
+# 输入限制
+# 图像格式：格式为jpg，jpeg，png，bmp，webp。
+
+# 图像分辨率：图像最小边长≥400像素，最大边长≤7000像素。
+
+# 图像坐标框：图像需先通过EMO图像检测API，以获得正确的人脸区域和动态区域坐标信息。
+
+# 音频格式：格式为wav、mp3。
+
+# 音频限制：文件<15M，时长＜60s。
+
+# 音频内容：音频中需包含清晰、响亮的人声语音，并去除了环境噪音、背景音乐等声音干扰信息。
+
+
+# 上传图片、音频链接仅支持HTTP链接方式，不支持本地链接方式。
+class DashscopeEmoCaller:
+    def __init__(self):
+        self.api_key = os.getenv("DASHSCOPE_API_KEY")
+        self.face_detect_url = "https://dashscope.aliyuncs.com/api/v1/services/aigc/image2video/face-detect"
+        self.video_systhesis_url = "https://dashscope.aliyuncs.com/api/v1/services/aigc/image2video/video-synthesis"
+
+    @classmethod
+    def INPUT_TYPES(cls):
+        return {
+            "required": {
+                "image": ("IMAGE", {"default": None}),
+                "audio": ("AUDIO", {"default": None}),
+            },
+        }
+
+    FUNCTION = "call_model"
+    OUTPUT_NODE = True
+    RETURN_TYPES = (
+        "IMAGE",
+        "INT",
+        "VHS_VIDEOINFO",
+        "AUDIO",
+    )
+    RETURN_NAMES = ("frames", "frame_count", "video_info", "audio")
+
+    CATEGORY = "dashscope"
+
+    def detect_face(self, image, ratio) -> tuple[str]:
+        ERROR_CODE_MAP = {
+            "InvalidParameter": "The request parameters format is invalid.",
+            "InvalidParameter.Ratio": "Invalid ratio parameter, must be '1:1' or '3:4'.",
+            "InvalidURL": "Failed to download input image, please check network or input format.",
+            "InvalidFile.NoHuman": "No human detected in the input image.",
+            "InvalidFile.MultiHuman": "Multiple humans detected in the image.",
+            "InvalidFile.BodyProportion": "The proportion of person in the image is invalid.",
+            "InvalidFile.Resolution": "Image resolution must be between 400px and 7000px.",
+            "InvalidFile.Value": "The image is too dark.",
+            "InvalidFile.FrontBody": "Person must be facing the camera.",
+            "InvalidFile.FullFace": "Face must be fully visible in the image.",
+            "InvalidFile.FacePose": "Face orientation has severe deviation.",
+            "InvalidFile.FullBody": "Full body or head must be completely visible based on ratio.",
+        }
+        ERROR_MESSAGE_MAP = {
+            "The input image has no human body. Please upload other image with single person.": "未检测到人脸。",
+            "The input image has multi human bodies. Please upload other image with single person.": "请上传单人照。",
+            "The proportion of the detected person in the picture is too large or too small, please upload other image.": "上传图片中人脸占比过大/过小。",
+            "The image resolution is invalid, please make sure that the largest length of image is smaller than 7000, and the smallest length of image is larger than 400.": "分辨率不得低于400*400。分辨率不得高于7000*7000。",
+            "The value of the image is invalid, please upload other clearer image.": "请确保图片中人脸清晰。",
+            "The pose of the detected person is invalid, please upload other image with the front view.": "请确保图片中人物正面朝向镜头。",
+            "The pose of the detected face is invalid, please upload other image with whole face.": "请确保图片中人脸完整无遮挡。",
+            "The pose of the detected face is invalid, please upload other image with the expected oriention.": "请确保图片中人脸朝向无偏斜。",
+            "The pose of the detected person is invalid, please upload other image with whole body, or change the ratio parameter to 1:1.": "请确保图片中人脸完整可见（针对1:1画幅）\n请确保图片中人物上半身完整可见（针对3:4画幅）。",
+        }
+
+        img_url = get_image_url(image)
+
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+        payload = {
+            "model": "emo-detect-v1",
+            "input": {"image_url": img_url},
+            "parameters": {"ratio": ratio},
+        }
+
+        response = requests.post(self.face_detect_url, headers=headers, json=payload)
+
+        if response.status_code == HTTPStatus.OK:
+            response_data = response.json()
+            output = response_data.get("output", {})
+            face_bbox = output.get("face_bbox", [])
+            ext_bbox = output.get("ext_bbox", [])
+            return (face_bbox, ext_bbox)
+        elif response.status_code == HTTPStatus.BAD_REQUEST:
+            error_response = response.json()
+            error_output = error_response.get("output", {})
+            error_code = error_output.get("code", "")
+            error_message = error_output.get("message", "")
+            error_message = ERROR_CODE_MAP.get(
+                error_code, f"Unknown error: {error_code}"
+            )
+            friendly_message = ERROR_MESSAGE_MAP.get(error_message, error_message)
+            print(friendly_message)
+            return (error_message,)
+
+    def _request_synthesis(self, image, audio):
+        img_url = get_image_url(image)
+        audio_url = get_audio_url(audio)
+
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+            "X-DashScope-Async": "enable",
+        }
+
+        payload = {
+            "model": "emo-v1",
+            "input": {"image_url": img_url, "audio_url": audio_url},
+            "parameters": {"style_level": "normal"},
+        }
+
+        response = requests.post(
+            self.video_systhesis_url, headers=headers, json=payload
+        )
+
+        if response.status_code != 200:
+            raise RuntimeError(f"Synthesis request failed: {response.text}")
+
+        return response.json()
+
+    def _check_task_status(self, task_id):
+        headers = {"Authorization": f"Bearer {self.api_key}"}
+
+        response = requests.get(
+            f"https://dashscope.aliyuncs.com/api/v1/tasks/{task_id}", headers=headers
+        )
+
+        if response.status_code != 200:
+            raise RuntimeError(f"Status check failed: {response.text}")
+
+        return response.json()
+
+    def call_model(self, image, audio):
+        if image is None or audio is None:
+            raise ValueError("Image and audio cannot be empty")
+
+        synthesis_response = self._request_synthesis(image, audio)
+        task_id = synthesis_response.get("output", {}).get("task_id")
+        if not task_id:
+            raise RuntimeError("Failed to get task_id")
+
+        while True:
+            status_response = self._check_task_status(task_id)
+            task_status = status_response.get("output", {}).get("task_status")
+
+            if task_status == "SUCCEEDED":
+                video_url = (
+                    status_response.get("output", {})
+                    .get("results", {})
+                    .get("video_url")
+                )
+                response = requests.get(video_url)
+                if response.status_code == 200:
+                    video_filename = generate_random_video_name()
+                    with open(video_filename, "wb") as f:
+                        f.write(response.content)
+                    frames, frame_count, video_info = video_to_images(video_filename)
+                    return (frames, frame_count, video_info, audio)
+                else:
+                    raise RuntimeError(
+                        f"Failed to download video: {response.status_code}"
+                    )
+            elif task_status == "FAILED":
+                error_message = status_response.get("output", {}).get(
+                    "message", "Unknown error"
+                )
+                raise RuntimeError(f"Task failed: {error_message}")
+            elif task_status in [
+                "PENDING",
+                "PRE-PROCESSING",
+                "RUNNING",
+                "POST-PROCESSING",
+            ]:
+                time.sleep(2)
+            else:
+                raise RuntimeError(f"Unknown task status: {task_status}")