From ce723cc0eb9d8d6fffec4fa338b9d7e37634f9e2 Mon Sep 17 00:00:00 2001 From: jakechai Date: Sun, 1 Feb 2026 21:50:24 +0800 Subject: [PATCH] v2.5.4 1. Update SAM3D Mesh Sequence From Video node: Align the mesh sequence to the video instead of aligning along the bbox center. --- README.md | 3 + nodes/jake_node_3d_sam3dseq.py | 425 ++++++++++++++++++++++----------- pyproject.toml | 2 +- 3 files changed, 295 insertions(+), 135 deletions(-) diff --git a/README.md b/README.md index ec05933..7e78d9b 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,9 @@ If you like what I share, please support me with [PayPal](https://paypal.me/jake - 2026-01-31 - v2.5.3 1. Add SAM3D Mesh Sequence From Video node to get mesh sequence from video using SAM3D. Need to install [ComfyUI-SAM3DBody](https://github.com/PozzettiAndrea/ComfyUI-SAM3DBody). +- 2026-02-01 - v2.5.4 +1. Update SAM3D Mesh Sequence From Video node: Align the mesh sequence to the video instead of aligning along the bbox center. + ## Installation 1. `git clone https://github.com/jakechai/ComfyUI-JakeUpgrade` into the `custom_nodes` folder - e.g. `custom_nodes\ComfyUI-JakeUpgrade` diff --git a/nodes/jake_node_3d_sam3dseq.py b/nodes/jake_node_3d_sam3dseq.py index f50f3c3..d8466e5 100644 --- a/nodes/jake_node_3d_sam3dseq.py +++ b/nodes/jake_node_3d_sam3dseq.py @@ -1,8 +1,6 @@ -""" -SAM3D Mesh Sequence Generator Nodes -处理视频、mhr_params和NPZ文件，生成二进制Mesh序列 (.bin) -""" - +#-----------------------------------------------------------------------------# +# Jake Upgrade SAM3D Mesh Sequence Generator Node for JK Custom Workflow of ComfyUI +#-----------------------------------------------------------------------------# import os import sys import time @@ -26,8 +24,57 @@ DEFAULT_FPS = 30.0 # 默认帧率 BINARY_MAGIC = b"MESH" # 魔数标识 BINARY_VERSION = 2 # 版本号 -# 获取当前文件所在目录 -CURRENT_FILE_DIR = Path(__file__).parent.absolute() +# ============================================================================= +# SAM3D 模型顶点和关节索引定义（仅备案未使用） +# ============================================================================= + +# 脚部顶点索引 +LEFT_FOOT_VERTICES = list(range(12223, 12234)) + list(range(12237, 13266)) # 12223-12233, 12237-13265 +RIGHT_FOOT_VERTICES = list(range(17396, 17407)) + list(range(17410, 18439)) # 17396-17406, 17410-18438 +FOOT_VERTICES = LEFT_FOOT_VERTICES + RIGHT_FOOT_VERTICES + +# 手部顶点索引 +LEFT_HAND_VERTICES = list(range(8883, 11234)) # 8883-11233 +RIGHT_HAND_VERTICES = list(range(14053, 14059)) + list(range(14063, 16407)) # 14053-14059, 14063-16406 + +# 脚踝关节索引 +LEFT_ANKLE_JOINT = 4 # joint_004 +RIGHT_ANKLE_JOINT = 20 # joint_020 +FOOT_JOINTS = [LEFT_ANKLE_JOINT, RIGHT_ANKLE_JOINT] + +# MHR127关节索引 +MHR127_JOINT_INDICES = { + "root": 0, # joint_000 + "pelvis": 1, # joint_001 + "spine1": 34, # joint_034 + "spine2": 35, # joint_035 + "spine3": 36, # joint_036 + "spine4": 37, # joint_037 + "neck": 110, # joint_110 + "head": 113, # joint_113 + "left_hip": 2, # joint_002 + "left_knee": 3, # joint_003 + "left_ankle": 4, # joint_004 + "left_heel": 6, # joint_006 + "left_foot": 7, # joint_007 + "left_toe": 8, # joint_008 + "right_hip": 18, # joint_018 + "right_knee": 19, # joint_019 + "right_ankle": 20, # joint_020 + "right_heel": 22, # joint_022 + "right_foot": 23, # joint_023 + "right_toe": 24, # joint_024 + "left_collar": 74, # joint_074 + "left_shoulder": 75, # joint_075 + "left_elbow": 76, # joint_076 + "left_wrist": 77, # joint_077 + "left_hand": 78, # joint_078 + "right_collar": 38, # joint_038 + "right_shoulder": 39, # joint_039 + "right_elbow": 40, # joint_040 + "right_wrist": 41, # joint_041 + "right_hand": 42, # joint_042 +} # ============================================================================= # 检查 SAM3D 依赖是否可用 @@ -39,39 +86,19 @@ def check_sam3d_dependencies(): 返回: (bool, str) - (是否可用, 错误信息) """ # 尝试多种可能的SAM3D节点路径 - possible_paths = [ - # 1. 尝试从当前文件向上回溯查找 - CURRENT_FILE_DIR.parent.parent.parent / "ComfyUI_Motion" / "custom_nodes" / "ComfyUI-SAM3DBody", - CURRENT_FILE_DIR.parent.parent.parent.parent / "ComfyUI_Motion" / "ComfyUI" / "custom_nodes" / "ComfyUI-SAM3DBody", - - # 2. 尝试ComfyUI根目录的相对路径 - CURRENT_FILE_DIR.parent.parent.parent / "ComfyUI-SAM3DBody", - - # 3. 尝试从环境变量获取 - os.environ.get("SAM3D_NODE_PATH", ""), - - # 4. 绝对路径备选 - Path(r"I:\ComfyUI_Motion\ComfyUI\custom_nodes\ComfyUI-SAM3DBody"), - ] + custom_nodes_dir = Path(__file__).absolute().parent.parent.parent + sam3d_node_path = custom_nodes_dir / "ComfyUI-SAM3DBody" - sam3d_node_path = None - for path_str in possible_paths: - if not path_str: - continue - - path = Path(path_str) if isinstance(path_str, str) else path_str - if path.exists(): - sam3d_node_path = path - print(f"[SAM3D Check] Found SAM3D node at: {sam3d_node_path}") - break + # 检查路径是否存在 + if not sam3d_node_path.exists(): + return False, "[SAM3D Check] SAM3D node path not found. Please ensure that the ComfyUI-SAM3DBody node is installed." - if sam3d_node_path is None: - return False, "未找到SAM3D节点路径，请确保已安装ComfyUI-SAM3DBody节点" + print(f"[SAM3D Check] ComfyUI-SAM3DBody path: {sam3d_node_path}") # 检查虚拟环境路径 sam3d_env_path = sam3d_node_path / "_env_sam3dbody" if not sam3d_env_path.exists(): - print(f"[SAM3D Check] Warning: SAM3D虚拟环境路径不存在: {sam3d_env_path}") + print(f"[SAM3D Check] Warning: The SAM3D virtual environment path does not exist: {sam3d_env_path}") # 虚拟环境不是必需的，可以继续 # 尝试导入 sam_3d_body @@ -83,12 +110,12 @@ def check_sam3d_dependencies(): # 尝试导入 from sam_3d_body import load_sam_3d_body, SAM3DBodyEstimator - print(f"[SAM3D Check] SAM3D依赖检查通过") - return True, "SAM3D依赖检查通过" + print(f"[SAM3D Check] SAM3D dependency check passed") + return True, "[SAM3D Check] SAM3D dependency check passed" except ImportError as e: - return False, f"无法导入SAM3D模块: {str(e)}" + return False, f"[SAM3D Check] Unable to import SAM3D module: {str(e)}" except Exception as e: - return False, f"SAM3D依赖检查失败: {str(e)}" + return False, f"[SAM3D Check] SAM3D dependency check failed: {str(e)}" # ============================================================================= # 二进制格式定义 @@ -177,16 +204,12 @@ class MeshSequenceBinaryFormat: 'file_size_mb': file_size / 1024 / 1024 } -# ============================================================================= -# 共享函数 -# ============================================================================= - # 模块级缓存，复用 SAM3DBodyProcess 中的模型 _MODEL_CACHE_PROCESS = {} def _load_sam3d_model_process(model_config: dict): """ - 加载 SAM 3D Body 模型（与 process.py 中的函数相同） + 加载 SAM 3D Body 模型 """ cache_key = model_config["ckpt_path"] @@ -196,7 +219,7 @@ def _load_sam3d_model_process(model_config: dict): # 检查依赖是否可用 available, error_msg = check_sam3d_dependencies() if not available: - raise ImportError(f"SAM3D依赖不可用: {error_msg}") + raise ImportError(f"[SAM3D Check] SAM3D dependency unavailable: {error_msg}") # 导入依赖 from sam_3d_body import load_sam_3d_body @@ -226,21 +249,7 @@ def _load_sam3d_model_process(model_config: dict): return result def apply_coordinate_transform(vertices_sequence: np.ndarray, transform_type: str = "rotate_z_180") -> np.ndarray: - """ - 对顶点序列应用坐标变换 - - 参数: - vertices_sequence: (num_frames, num_verts, 3) 的顶点序列 - transform_type: 变换类型 - - "rotate_z_180": 绕Z轴旋转180度 (X,Y取反) - - "rotate_y_180": 绕Y轴旋转180度 (X,Z取反) - - "rotate_x_180": 绕X轴旋转180度 (Y,Z取反) - - "rotate_z_90": 绕Z轴旋转90度 - - "none": 不应用变换 - - 返回: - 变换后的顶点序列 - """ + if transform_type == "none" or vertices_sequence is None: return vertices_sequence @@ -249,30 +258,19 @@ def apply_coordinate_transform(vertices_sequence: np.ndarray, transform_type: st if transform_type == "rotate_z_180": # 绕Z轴旋转180度: (x, y, z) -> (-x, -y, z) - print(f"[CoordinateTransform] Rotate 180 degrees around Z axis: {num_frames}frames, {num_verts}vertices") transformed_sequence[:, :, 0] = -vertices_sequence[:, :, 0] # X取反 transformed_sequence[:, :, 1] = -vertices_sequence[:, :, 1] # Y取反 # Z保持不变 elif transform_type == "rotate_y_180": # 绕Y轴旋转180度: (x, y, z) -> (-x, y, -z) - print(f"[CoordinateTransform] Rotate 180 degrees around Y axis: {num_frames}frames, {num_verts}vertices") transformed_sequence[:, :, 0] = -vertices_sequence[:, :, 0] # X取反 transformed_sequence[:, :, 2] = -vertices_sequence[:, :, 2] # Z取反 # Y保持不变 elif transform_type == "rotate_x_180": # 绕X轴旋转180度: (x, y, z) -> (x, -y, -z) - print(f"[CoordinateTransform] Rotate 180 degrees around X axis: {num_frames}frames, {num_verts}vertices") transformed_sequence[:, :, 1] = -vertices_sequence[:, :, 1] # Y取反 transformed_sequence[:, :, 2] = -vertices_sequence[:, :, 2] # Z取反 # X保持不变 - elif transform_type == "rotate_z_90": - # 绕Z轴旋转90度: (x, y, z) -> (-y, x, z) - print(f"[CoordinateTransform] Rotate 90 degrees around Z axis: {num_frames}frames, {num_verts}vertices") - x_original = vertices_sequence[:, :, 0].copy() - y_original = vertices_sequence[:, :, 1].copy() - transformed_sequence[:, :, 0] = -y_original # X = -Y - transformed_sequence[:, :, 1] = x_original # Y = X - # Z保持不变 else: print(f"[CoordinateTransform] Unknown transformation type: {transform_type}, no transformation applied") @@ -371,38 +369,137 @@ def moving_average_smooth_numpy(x_np, window_size=5, dim=0): else: return x +def align_models_to_fixed_camera(frames_data, faces, reference_frame=0): + """ + 将所有模型对齐到固定相机（参考帧的相机） + + 参数： + - frames_data: 每帧的数据列表，包含vertices, cam_t, focal_length + - faces: 面数据 + - reference_frame: 参考帧索引（通常为0） + + 返回： + - aligned_vertices: 对齐后的顶点序列 + - valid_frames: 有效帧的索引列表 + - reference_cam_params: 参考帧的相机参数 + """ + + # 找到第一个有效帧作为参考帧（如果指定参考帧无效） + valid_indices = [i for i, d in enumerate(frames_data) if d["success"]] + if not valid_indices: + raise ValueError("[Alignment] No valid frames found") + + # 如果指定的参考帧无效，使用第一个有效帧 + if not frames_data[reference_frame]["success"]: + reference_frame = valid_indices[0] + print(f"[Alignment] Reference frame changed to first valid frame: {reference_frame}") + + # 获取参考帧数据 + ref_data = frames_data[reference_frame] + ref_vertices = ref_data["vertices"] + ref_cam_t = ref_data["cam_t"] + ref_focal_length = ref_data["focal_length"] + + if ref_vertices is None or ref_cam_t is None: + raise ValueError(f"[Alignment] Reference frame {reference_frame} has invalid data") + + print(f"[Alignment] Reference frame: {reference_frame}") + print(f"[Alignment] Reference camera: cam_t={ref_cam_t}, focal={ref_focal_length}") + + # 准备输出序列 + num_frames = len(frames_data) + num_verts = ref_vertices.shape[0] + aligned_vertices = np.zeros((num_frames, num_verts, 3)) + valid_frames = [] + + # 处理每一帧 + for i in range(num_frames): + frame_data = frames_data[i] + + if not frame_data["success"]: + # 无效帧：复制参考帧的顶点（或保持为0） + aligned_vertices[i] = ref_vertices.copy() + continue + + vertices = frame_data["vertices"] + cam_t = frame_data["cam_t"] + focal_length = frame_data["focal_length"] + + if vertices is None or cam_t is None: + # 数据不完整 + aligned_vertices[i] = ref_vertices.copy() + continue + + # 检查形状一致性 + if vertices.shape[0] != num_verts: + print(f"[Alignment] Warning: Frame {i} has {vertices.shape[0]} vertices, expected {num_verts}") + aligned_vertices[i] = ref_vertices.copy() + continue + + # 应用对齐变换：V_i' = V_i + C_i - C_ref + # 注意：这里假设焦距相同，如果需要处理变焦，需要额外的缩放 + if focal_length is not None and ref_focal_length is not None and focal_length != ref_focal_length: + # 如果需要处理焦距变化 + scale = ref_focal_length / focal_length + aligned_vertices[i] = (vertices + cam_t) * scale - ref_cam_t + if i < 3: # 打印前几帧的调试信息 + print(f"[Alignment] Frame {i}: scaled by {scale:.3f} (focal: {focal_length:.1f} -> {ref_focal_length:.1f})") + else: + # 简单的相机位置对齐 + aligned_vertices[i] = vertices + cam_t - ref_cam_t + + valid_frames.append(i) + + # 调试信息 + if i < 3: # 只打印前3帧的详细信息 + print(f"[Alignment] Frame {i}: cam_t={cam_t.flatten()}, aligned offset={-cam_t.flatten() + ref_cam_t.flatten()}") + + print(f"[Alignment] Completed: {len(valid_frames)} valid frames aligned to reference camera") + + # 计算对齐后的统计信息 + if valid_frames: + # 计算有效帧的顶点变化范围 + valid_vertices = aligned_vertices[valid_frames] + min_pos = valid_vertices.min(axis=(0, 1)) + max_pos = valid_vertices.max(axis=(0, 1)) + range_pos = max_pos - min_pos + + print(f"[Alignment] Vertex position range after alignment:") + print(f" X: {min_pos[0]:.3f} to {max_pos[0]:.3f} (range: {range_pos[0]:.3f})") + print(f" Y: {min_pos[1]:.3f} to {max_pos[1]:.3f} (range: {range_pos[1]:.3f})") + print(f" Z: {min_pos[2]:.3f} to {max_pos[2]:.3f} (range: {range_pos[2]:.3f})") + + return aligned_vertices, valid_frames, { + "cam_t": ref_cam_t, + "focal_length": ref_focal_length, + "reference_frame": reference_frame + } # ============================================================================= # 视频处理节点 # ============================================================================= class SAM3DMeshSequenceFromVideo_JK: - """ - 从视频文件生成 Mesh 序列，使用与 SAM3DBodyProcess 相同的输入格式 - - 注意: 此节点需要 SAM3DBody 节点的依赖 - """ @classmethod def INPUT_TYPES(cls): # 检查依赖是否可用 available, error_msg = check_sam3d_dependencies() if not available: - print(f"警告: SAM3D依赖不可用，节点可能无法正常工作: {error_msg}") + print(f"Warning: ComfyUI-SAM3DBody dependency is unavailable; nodes may not function correctly: {error_msg}") return { "required": { "model": ("SAM3D_MODEL", { - "tooltip": "SAM3D 模型配置" + "tooltip": "SAM3D Model" }), "image": ("IMAGE", { - "tooltip": "视频帧序列（批处理的图像）" + "tooltip": "Image Batch" }), "output_filename": ("STRING", { - "default": "mesh_sequence.bin", + "default": "mesh_sequence", "multiline": False, - "placeholder": "输出文件名 (会自动添加时间戳)", - "tooltip": "输出二进制文件名，会保存在 Adv3DViewer_JK_tmp 目录" + "tooltip": "The output binary filename will be saved in the Adv3DViewer_JK_tmp directory of the output folder." }), }, "optional": { @@ -411,30 +508,43 @@ class SAM3DMeshSequenceFromVideo_JK: "min": 0.0, "max": 1.0, "step": 0.05, - "tooltip": "人体检测阈值" + "tooltip": "Human body detection threshold" }), "inference_type": (["full", "body", "hand"], { "default": "full", - "tooltip": "推理类型" + "tooltip": "Inference type" }), "mask": ("MASK", { - "tooltip": "可选的分割掩码序列" + "tooltip": "Optional Mask" }), - "coordinate_transform": (["none", "rotate_z_180", "rotate_y_180", "rotate_x_180", "rotate_z_90"], { - "default": "rotate_z_180", - "tooltip": "坐标变换类型。通常使用rotate_z_180修正朝向问题" + # "coordinate_transform": (["none", "rotate_z_180", "rotate_y_180", "rotate_x_180"], { + # "default": "rotate_z_180", + # "tooltip": "Coordinate transformation type. Rotate_z_180 is typically used to correct SAM3D Body orientation issues." + # }), + "smoothing_method": (["gaussian", "moving_average"], { + "default": "gaussian", + "tooltip": "Smoothing Method" }), "smoothing_sigma": ("FLOAT", { "default": 3.0, "min": 0.5, "max": 10.0, "step": 0.5, - "tooltip": "高斯平滑核宽度（越高越平滑）" - }), - "smoothing_method": (["gaussian", "moving_average"], { - "default": "gaussian", - "tooltip": "平滑算法" + "tooltip": "Gaussian smoothing kernel width (the higher the width, the smoother the surface)" }), + # "align_to_reference_camera": ("BOOLEAN", { + # "default": True, + # "label_on": "Enabled", + # "label_off": "Disabled", + # "tooltip": "Align all models to the camera (fixed camera) of the reference frame to ensure perfect alignment with the video." + # }), + # "reference_frame": ("INT", { + # "default": 0, + # "min": 0, + # "max": 1000, + # "step": 1, + # "tooltip": "Reference frame index (used for camera alignment)" + # }), } } @@ -449,7 +559,7 @@ class SAM3DMeshSequenceFromVideo_JK: # 检查依赖是否可用 available, error_msg = check_sam3d_dependencies() if not available: - print(f"警告: SAM3D依赖不可用，节点可能无法正常工作: {error_msg}") + print(f"[SAM3D Check] ComfyUI-SAM3DBody dependency is unavailable; nodes may not function correctly: {error_msg}") import folder_paths self.output_dir = folder_paths.get_output_directory() @@ -459,8 +569,6 @@ class SAM3DMeshSequenceFromVideo_JK: tmp_output_dir = Path(self.output_dir) / self.tmp_output_dir_name tmp_output_dir.mkdir(parents=True, exist_ok=True) - print(f"[SAM3DMeshSequence] Output directory: {tmp_output_dir}") - def get_tmp_output_dir(self): """获取临时输出目录的 Path 对象""" return Path(self.output_dir) / self.tmp_output_dir_name @@ -496,27 +604,21 @@ class SAM3DMeshSequenceFromVideo_JK: def generate_from_video_frames(self, model, image, output_filename, bbox_threshold=0.8, inference_type="full", mask=None, coordinate_transform="rotate_z_180", - smoothing_sigma=3.0, smoothing_method="gaussian"): - """ - 从图像序列（视频帧）生成 Mesh 序列 - """ + smoothing_sigma=3.0, smoothing_method="gaussian", + align_to_reference_camera=True, reference_frame=0): - import time start_time_total = time.time() # 检查依赖是否可用 available, error_msg = check_sam3d_dependencies() if not available: - return (f"错误: SAM3D依赖不可用 - {error_msg}",) + return (f"[SAM3D Check] SAM3D dependency unavailable: {error_msg}",) # 1. 检查输入图像 if image is None or len(image) == 0: - return (f"错误: 没有输入图像",) + return (f"[VideoFramesToMesh] Error: No input image",) num_frames = len(image) - print(f"[VideoFramesToMesh] Input image sequence: {num_frames}frames") - print(f"[VideoFramesToMesh] Coordinate transformation: {coordinate_transform}") - print(f"[VideoFramesToMesh] Temporal smoothing: enabled, method: {smoothing_method}, sigma: {smoothing_sigma}") # 2. 加载模型 model_start_time = time.time() @@ -531,9 +633,7 @@ class SAM3DMeshSequenceFromVideo_JK: try: loaded = _load_sam3d_model_process(model_config) except ImportError as e: - return (f"错误: 无法加载SAM3D模型 - {str(e)}",) - - print(f"[VideoFramesToMesh] Model loaded, time: {time.time() - model_start_time:.2f}seconds") + return (f"[SAM3D Check] SAM3D dependency unavailable: {str(e)}",) # 3. 获取面数据 from sam_3d_body import SAM3DBodyEstimator @@ -557,7 +657,7 @@ class SAM3DMeshSequenceFromVideo_JK: pass if faces is None: - return (f"错误: 无法获取面数据",) + return (f"[VideoFramesToMesh] Error: Unable to retrieve face data",) # 4. 准备输出路径 # 清理文件名 @@ -579,10 +679,8 @@ class SAM3DMeshSequenceFromVideo_JK: tmp_output_dir = self.get_tmp_output_dir() output_path = tmp_output_dir / unique_filename - print(f"[VideoFramesToMesh] Output file: {output_path}") - # 5. 处理图像序列 - 收集顶点信息 - vertices_sequence = [] + frames_data = [] # 存储每帧的完整数据 processed_frames = 0 failed_frames = 0 @@ -644,11 +742,13 @@ class SAM3DMeshSequenceFromVideo_JK: print(f"[VideoFramesToMesh] Warning: Frame {frame_idx} no human detected") failed_frames += 1 - # 添加空白帧 - num_verts = faces.shape[0] - blank_vertices = np.zeros((num_verts, 3)) - vertices_sequence.append(blank_vertices) - + # 添加空白帧数据 + frames_data.append({ + "vertices": None, + "cam_t": None, + "focal_length": None, + "success": False + }) processed_frames += 1 continue @@ -657,11 +757,19 @@ class SAM3DMeshSequenceFromVideo_JK: # 提取顶点 pred_vertices = output.get("pred_vertices") + + # 提取相机参数 + pred_cam_t = output.get("pred_cam_t") + focal_length = output.get("focal_length") + if pred_vertices is None: print(f"[VideoFramesToMesh] Warning: Frame {frame_idx} no vertex output, using blank frame") - num_verts = faces.shape[0] - blank_vertices = np.zeros((num_verts, 3)) - vertices_sequence.append(blank_vertices) + frames_data.append({ + "vertices": None, + "cam_t": None, + "focal_length": None, + "success": False + }) else: # 转换为 numpy 数组 if torch.is_tensor(pred_vertices): @@ -669,7 +777,24 @@ class SAM3DMeshSequenceFromVideo_JK: else: vertices = pred_vertices - vertices_sequence.append(vertices) + # 处理相机参数 + if torch.is_tensor(pred_cam_t): + cam_t = pred_cam_t.detach().cpu().numpy() + else: + cam_t = pred_cam_t + + if focal_length is not None: + if hasattr(focal_length, 'item'): + focal_length = focal_length.item() + focal_length = float(focal_length) + + # 保存帧数据 + frames_data.append({ + "vertices": vertices, + "cam_t": cam_t, + "focal_length": focal_length, + "success": True + }) processed_frames += 1 @@ -677,10 +802,12 @@ class SAM3DMeshSequenceFromVideo_JK: print(f"[VideoFramesToMesh] Error: Processing frame {frame_idx} failed - {e}") failed_frames += 1 - # 添加空白帧 - num_verts = faces.shape[0] - blank_vertices = np.zeros((num_verts, 3)) - vertices_sequence.append(blank_vertices) + frames_data.append({ + "vertices": None, + "cam_t": None, + "focal_length": None, + "success": False + }) processed_frames += 1 @@ -695,18 +822,47 @@ class SAM3DMeshSequenceFromVideo_JK: if processed_frames == 0: return (f"错误: 没有成功处理任何帧",) - # 6. 转换为 numpy 数组 - vertices_sequence_np = np.stack(vertices_sequence, axis=0) + # 6. 根据参数决定是否应用模型对齐 + if align_to_reference_camera: + print(f"[VideoFramesToMesh] Aligning models to fixed camera (reference frame: {reference_frame})...") + try: + aligned_vertices, valid_frames, ref_cam_params = align_models_to_fixed_camera( + frames_data, + faces, + reference_frame=reference_frame + ) + + print(f"[Alignment] Reference camera parameters:") + print(f" cam_t: {ref_cam_params['cam_t'].flatten()}") + print(f" focal_length: {ref_cam_params['focal_length']}") + + except Exception as e: + print(f"[Alignment] Error during alignment: {e}") + print(f"[Alignment] Falling back to original vertices") + + # 回退：使用原始顶点（不应用对齐） + aligned_vertices = np.stack([d["vertices"] if d["success"] else np.zeros((faces.shape[0], 3)) + for d in frames_data], axis=0) + valid_frames = [i for i, d in enumerate(frames_data) if d["success"]] + else: + print(f"[VideoFramesToMesh] Skipping model alignment (align_to_reference_camera=False)") + # 使用原始顶点 + aligned_vertices = np.stack([d["vertices"] if d["success"] else np.zeros((faces.shape[0], 3)) + for d in frames_data], axis=0) + valid_frames = [i for i, d in enumerate(frames_data) if d["success"]] # 7. 总是应用时间平滑（如果有多于1帧） - if processed_frames > 1: + if len(valid_frames) > 1: print(f"[VideoFramesToMesh] Applying temporal smoothing (method: {smoothing_method}, sigma: {smoothing_sigma})") - vertices_sequence_np = self.smooth_sequence( - vertices_sequence_np, + # 只对有效帧应用平滑 + valid_vertices = aligned_vertices[valid_frames] + smoothed_vertices = self.smooth_sequence( + valid_vertices, sigma=smoothing_sigma, method=smoothing_method ) + aligned_vertices[valid_frames] = smoothed_vertices print(f"[VideoFramesToMesh] Temporal smoothing completed") @@ -715,19 +871,20 @@ class SAM3DMeshSequenceFromVideo_JK: print(f"[CoordinateTransform] Applying coordinate transformation: {coordinate_transform}") # 变换顶点序列 - vertices_sequence_np = apply_coordinate_transform(vertices_sequence_np, coordinate_transform) + aligned_vertices = apply_coordinate_transform(aligned_vertices, coordinate_transform) # 9. 总是保存为 SMPL 兼容格式 MeshSequenceBinaryFormat.save_smpl_compatible( - vertices_sequence=vertices_sequence_np, + vertices_sequence=aligned_vertices, faces=faces, output_path=output_path, fps=DEFAULT_FPS, coordinate_transform="none" # 已经在前面应用了变换 ) - print(f"[VideoFramesToMesh] SMPL compatible format generation completed! Total time: {time.time() - start_time_total:.2f}seconds") - print(f"[VideoFramesToMesh] Successfully processed: {processed_frames - failed_frames}frames, Failed: {failed_frames}frames") + print(f"[VideoFramesToMesh] Model alignment and generation completed!") + print(f"[VideoFramesToMesh] Successfully processed: {processed_frames - failed_frames} frames, Failed: {failed_frames} frames") + print(f"[VideoFramesToMesh] Model alignment: enabled (reference frame: {ref_cam_params.get('reference_frame', 0)})") print(f"[VideoFramesToMesh] Coordinate transformation: {coordinate_transform}") print(f"[VideoFramesToMesh] Temporal smoothing: enabled ({smoothing_method}, sigma={smoothing_sigma})") print(f"[VideoFramesToMesh] Output file: {output_path}") diff --git a/pyproject.toml b/pyproject.toml index 009782b..8cd5e6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-jakeupgrade" description = "ComfyUI workflow customization by Jake." -version = "2.5.3" +version = "2.5.4" license = {file = "LICENSE"} dependencies = ["opencv-python", "piexif", "torch", "torchvision", "numpy", "pillow", "simpleeval", "PyYAML", "toml"]