nodes/jakechai-ComfyUI-JakeUpgrade
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v2.5.4

Browse Source 
1. Update SAM3D Mesh Sequence From Video node: Align the mesh sequence to the video instead of aligning along the bbox center.
This commit is contained in:
jakechai
2026-02-01 21:50:24 +08:00
parent d3f07e8597
commit ce723cc0eb
3 changed files with 295 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3
View File
@@ -399,6 +399,9 @@ If you like what I share, please support me with [PayPal](https://paypal.me/jake
- 2026-01-31 - v2.5.3
1. Add SAM3D Mesh Sequence From Video node to get mesh sequence from video using SAM3D. Need to install [ComfyUI-SAM3DBody](https://github.com/PozzettiAndrea/ComfyUI-SAM3DBody).
- 2026-02-01 - v2.5.4
1. Update SAM3D Mesh Sequence From Video node: Align the mesh sequence to the video instead of aligning along the bbox center.
## Installation
1. `git clone https://github.com/jakechai/ComfyUI-JakeUpgrade` into the `custom_nodes` folder
- e.g. `custom_nodes\ComfyUI-JakeUpgrade`
nodes/jake_node_3d_sam3dseq.py
+291 -134
View File
@@ -1,8 +1,6 @@
"""
SAM3D Mesh Sequence Generator Nodes
处理视频、mhr_params和NPZ文件，生成二进制Mesh序列 (.bin)
"""
#-----------------------------------------------------------------------------#
# Jake Upgrade SAM3D Mesh Sequence Generator Node for JK Custom Workflow of ComfyUI
#-----------------------------------------------------------------------------#
import os
import sys
import time
@@ -26,8 +24,57 @@ DEFAULT_FPS = 30.0 # 默认帧率
BINARY_MAGIC = b"MESH" # 魔数标识
BINARY_VERSION = 2 # 版本号
# 获取当前文件所在目录
CURRENT_FILE_DIR = Path(__file__).parent.absolute()
# =============================================================================
# SAM3D 模型顶点和关节索引定义（仅备案未使用）
# =============================================================================
# 脚部顶点索引
LEFT_FOOT_VERTICES = list(range(12223, 12234)) + list(range(12237, 13266)) # 12223-12233, 12237-13265
RIGHT_FOOT_VERTICES = list(range(17396, 17407)) + list(range(17410, 18439)) # 17396-17406, 17410-18438
FOOT_VERTICES = LEFT_FOOT_VERTICES + RIGHT_FOOT_VERTICES
# 手部顶点索引
LEFT_HAND_VERTICES = list(range(8883, 11234)) # 8883-11233
RIGHT_HAND_VERTICES = list(range(14053, 14059)) + list(range(14063, 16407)) # 14053-14059, 14063-16406
# 脚踝关节索引
LEFT_ANKLE_JOINT = 4 # joint_004
RIGHT_ANKLE_JOINT = 20 # joint_020
FOOT_JOINTS = [LEFT_ANKLE_JOINT, RIGHT_ANKLE_JOINT]
# MHR127关节索引
MHR127_JOINT_INDICES = {
"root": 0, # joint_000
"pelvis": 1, # joint_001
"spine1": 34, # joint_034
"spine2": 35, # joint_035
"spine3": 36, # joint_036
"spine4": 37, # joint_037
"neck": 110, # joint_110
"head": 113, # joint_113
"left_hip": 2, # joint_002
"left_knee": 3, # joint_003
"left_ankle": 4, # joint_004
"left_heel": 6, # joint_006
"left_foot": 7, # joint_007
"left_toe": 8, # joint_008
"right_hip": 18, # joint_018
"right_knee": 19, # joint_019
"right_ankle": 20, # joint_020
"right_heel": 22, # joint_022
"right_foot": 23, # joint_023
"right_toe": 24, # joint_024
"left_collar": 74, # joint_074
"left_shoulder": 75, # joint_075
"left_elbow": 76, # joint_076
"left_wrist": 77, # joint_077
"left_hand": 78, # joint_078
"right_collar": 38, # joint_038
"right_shoulder": 39, # joint_039
"right_elbow": 40, # joint_040
"right_wrist": 41, # joint_041
"right_hand": 42, # joint_042
}
# =============================================================================
# 检查 SAM3D 依赖是否可用
@@ -39,39 +86,19 @@ def check_sam3d_dependencies():
返回: (bool, str) - (是否可用, 错误信息)
"""
# 尝试多种可能的SAM3D节点路径
possible_paths = [
# 1. 尝试从当前文件向上回溯查找
CURRENT_FILE_DIR.parent.parent.parent / "ComfyUI_Motion" / "custom_nodes" / "ComfyUI-SAM3DBody",
CURRENT_FILE_DIR.parent.parent.parent.parent / "ComfyUI_Motion" / "ComfyUI" / "custom_nodes" / "ComfyUI-SAM3DBody",
# 2. 尝试ComfyUI根目录的相对路径
CURRENT_FILE_DIR.parent.parent.parent / "ComfyUI-SAM3DBody",
# 3. 尝试从环境变量获取
os.environ.get("SAM3D_NODE_PATH", ""),
# 4. 绝对路径备选
Path(r"I:\ComfyUI_Motion\ComfyUI\custom_nodes\ComfyUI-SAM3DBody"),
]
custom_nodes_dir = Path(__file__).absolute().parent.parent.parent
sam3d_node_path = custom_nodes_dir / "ComfyUI-SAM3DBody"
sam3d_node_path = None
for path_str in possible_paths:
if not path_str:
continue
path = Path(path_str) if isinstance(path_str, str) else path_str
if path.exists():
sam3d_node_path = path
print(f"[SAM3D Check] Found SAM3D node at: {sam3d_node_path}")
break
# 检查路径是否存在
if not sam3d_node_path.exists():
return False, "[SAM3D Check] SAM3D node path not found. Please ensure that the ComfyUI-SAM3DBody node is installed."
if sam3d_node_path is None:
return False, "未找到SAM3D节点路径，请确保已安装ComfyUI-SAM3DBody节点"
print(f"[SAM3D Check] ComfyUI-SAM3DBody path: {sam3d_node_path}")
# 检查虚拟环境路径
sam3d_env_path = sam3d_node_path / "_env_sam3dbody"
if not sam3d_env_path.exists():
print(f"[SAM3D Check] Warning: SAM3D虚拟环境路径不存在: {sam3d_env_path}")
print(f"[SAM3D Check] Warning: The SAM3D virtual environment path does not exist: {sam3d_env_path}")
# 虚拟环境不是必需的，可以继续
# 尝试导入 sam_3d_body
@@ -83,12 +110,12 @@ def check_sam3d_dependencies():
# 尝试导入
from sam_3d_body import load_sam_3d_body, SAM3DBodyEstimator
print(f"[SAM3D Check] SAM3D依赖检查通过")
return True, "SAM3D依赖检查通过"
print(f"[SAM3D Check] SAM3D dependency check passed")
return True, "[SAM3D Check] SAM3D dependency check passed"
except ImportError as e:
return False, f"无法导入SAM3D模块: {str(e)}"
return False, f"[SAM3D Check] Unable to import SAM3D module: {str(e)}"
except Exception as e:
return False, f"SAM3D依赖检查失败: {str(e)}"
return False, f"[SAM3D Check] SAM3D dependency check failed: {str(e)}"
# =============================================================================
# 二进制格式定义
@@ -177,16 +204,12 @@ class MeshSequenceBinaryFormat:
'file_size_mb': file_size / 1024 / 1024
}
# =============================================================================
# 共享函数
# =============================================================================
# 模块级缓存，复用 SAM3DBodyProcess 中的模型
_MODEL_CACHE_PROCESS = {}
def _load_sam3d_model_process(model_config: dict):
"""
加载 SAM 3D Body 模型（与 process.py 中的函数相同）
加载 SAM 3D Body 模型
"""
cache_key = model_config["ckpt_path"]
@@ -196,7 +219,7 @@ def _load_sam3d_model_process(model_config: dict):
# 检查依赖是否可用
available, error_msg = check_sam3d_dependencies()
if not available:
raise ImportError(f"SAM3D依赖不可用: {error_msg}")
raise ImportError(f"[SAM3D Check] SAM3D dependency unavailable: {error_msg}")
# 导入依赖
from sam_3d_body import load_sam_3d_body
@@ -226,21 +249,7 @@ def _load_sam3d_model_process(model_config: dict):
return result
def apply_coordinate_transform(vertices_sequence: np.ndarray, transform_type: str = "rotate_z_180") -> np.ndarray:
"""
对顶点序列应用坐标变换
参数:
vertices_sequence: (num_frames, num_verts, 3) 的顶点序列
transform_type: 变换类型
- "rotate_z_180": 绕Z轴旋转180度 (X,Y取反)
- "rotate_y_180": 绕Y轴旋转180度 (X,Z取反)
- "rotate_x_180": 绕X轴旋转180度 (Y,Z取反)
- "rotate_z_90": 绕Z轴旋转90度
- "none": 不应用变换
返回:
变换后的顶点序列
"""
if transform_type == "none" or vertices_sequence is None:
return vertices_sequence
@@ -249,30 +258,19 @@ def apply_coordinate_transform(vertices_sequence: np.ndarray, transform_type: st
if transform_type == "rotate_z_180":
# 绕Z轴旋转180度: (x, y, z) -> (-x, -y, z)
print(f"[CoordinateTransform] Rotate 180 degrees around Z axis: {num_frames}frames, {num_verts}vertices")
transformed_sequence[:, :, 0] = -vertices_sequence[:, :, 0] # X取反
transformed_sequence[:, :, 1] = -vertices_sequence[:, :, 1] # Y取反
# Z保持不变
elif transform_type == "rotate_y_180":
# 绕Y轴旋转180度: (x, y, z) -> (-x, y, -z)
print(f"[CoordinateTransform] Rotate 180 degrees around Y axis: {num_frames}frames, {num_verts}vertices")
transformed_sequence[:, :, 0] = -vertices_sequence[:, :, 0] # X取反
transformed_sequence[:, :, 2] = -vertices_sequence[:, :, 2] # Z取反
# Y保持不变
elif transform_type == "rotate_x_180":
# 绕X轴旋转180度: (x, y, z) -> (x, -y, -z)
print(f"[CoordinateTransform] Rotate 180 degrees around X axis: {num_frames}frames, {num_verts}vertices")
transformed_sequence[:, :, 1] = -vertices_sequence[:, :, 1] # Y取反
transformed_sequence[:, :, 2] = -vertices_sequence[:, :, 2] # Z取反
# X保持不变
elif transform_type == "rotate_z_90":
# 绕Z轴旋转90度: (x, y, z) -> (-y, x, z)
print(f"[CoordinateTransform] Rotate 90 degrees around Z axis: {num_frames}frames, {num_verts}vertices")
x_original = vertices_sequence[:, :, 0].copy()
y_original = vertices_sequence[:, :, 1].copy()
transformed_sequence[:, :, 0] = -y_original # X = -Y
transformed_sequence[:, :, 1] = x_original # Y = X
# Z保持不变
else:
print(f"[CoordinateTransform] Unknown transformation type: {transform_type}, no transformation applied")
@@ -371,38 +369,137 @@ def moving_average_smooth_numpy(x_np, window_size=5, dim=0):
else:
return x
def align_models_to_fixed_camera(frames_data, faces, reference_frame=0):
"""
将所有模型对齐到固定相机（参考帧的相机）
参数：
- frames_data: 每帧的数据列表，包含vertices, cam_t, focal_length
- faces: 面数据
- reference_frame: 参考帧索引（通常为0）
返回：
- aligned_vertices: 对齐后的顶点序列
- valid_frames: 有效帧的索引列表
- reference_cam_params: 参考帧的相机参数
"""
# 找到第一个有效帧作为参考帧（如果指定参考帧无效）
valid_indices = [i for i, d in enumerate(frames_data) if d["success"]]
if not valid_indices:
raise ValueError("[Alignment] No valid frames found")
# 如果指定的参考帧无效，使用第一个有效帧
if not frames_data[reference_frame]["success"]:
reference_frame = valid_indices[0]
print(f"[Alignment] Reference frame changed to first valid frame: {reference_frame}")
# 获取参考帧数据
ref_data = frames_data[reference_frame]
ref_vertices = ref_data["vertices"]
ref_cam_t = ref_data["cam_t"]
ref_focal_length = ref_data["focal_length"]
if ref_vertices is None or ref_cam_t is None:
raise ValueError(f"[Alignment] Reference frame {reference_frame} has invalid data")
print(f"[Alignment] Reference frame: {reference_frame}")
print(f"[Alignment] Reference camera: cam_t={ref_cam_t}, focal={ref_focal_length}")
# 准备输出序列
num_frames = len(frames_data)
num_verts = ref_vertices.shape[0]
aligned_vertices = np.zeros((num_frames, num_verts, 3))
valid_frames = []
# 处理每一帧
for i in range(num_frames):
frame_data = frames_data[i]
if not frame_data["success"]:
# 无效帧：复制参考帧的顶点（或保持为0）
aligned_vertices[i] = ref_vertices.copy()
continue
vertices = frame_data["vertices"]
cam_t = frame_data["cam_t"]
focal_length = frame_data["focal_length"]
if vertices is None or cam_t is None:
# 数据不完整
aligned_vertices[i] = ref_vertices.copy()
continue
# 检查形状一致性
if vertices.shape[0] != num_verts:
print(f"[Alignment] Warning: Frame {i} has {vertices.shape[0]} vertices, expected {num_verts}")
aligned_vertices[i] = ref_vertices.copy()
continue
# 应用对齐变换：V_i' = V_i + C_i - C_ref
# 注意：这里假设焦距相同，如果需要处理变焦，需要额外的缩放
if focal_length is not None and ref_focal_length is not None and focal_length != ref_focal_length:
# 如果需要处理焦距变化
scale = ref_focal_length / focal_length
aligned_vertices[i] = (vertices + cam_t) * scale - ref_cam_t
if i < 3: # 打印前几帧的调试信息
print(f"[Alignment] Frame {i}: scaled by {scale:.3f} (focal: {focal_length:.1f} -> {ref_focal_length:.1f})")
else:
# 简单的相机位置对齐
aligned_vertices[i] = vertices + cam_t - ref_cam_t
valid_frames.append(i)
# 调试信息
if i < 3: # 只打印前3帧的详细信息
print(f"[Alignment] Frame {i}: cam_t={cam_t.flatten()}, aligned offset={-cam_t.flatten() + ref_cam_t.flatten()}")
print(f"[Alignment] Completed: {len(valid_frames)} valid frames aligned to reference camera")
# 计算对齐后的统计信息
if valid_frames:
# 计算有效帧的顶点变化范围
valid_vertices = aligned_vertices[valid_frames]
min_pos = valid_vertices.min(axis=(0, 1))
max_pos = valid_vertices.max(axis=(0, 1))
range_pos = max_pos - min_pos
print(f"[Alignment] Vertex position range after alignment:")
print(f" X: {min_pos[0]:.3f} to {max_pos[0]:.3f} (range: {range_pos[0]:.3f})")
print(f" Y: {min_pos[1]:.3f} to {max_pos[1]:.3f} (range: {range_pos[1]:.3f})")
print(f" Z: {min_pos[2]:.3f} to {max_pos[2]:.3f} (range: {range_pos[2]:.3f})")
return aligned_vertices, valid_frames, {
"cam_t": ref_cam_t,
"focal_length": ref_focal_length,
"reference_frame": reference_frame
}
# =============================================================================
# 视频处理节点
# =============================================================================
class SAM3DMeshSequenceFromVideo_JK:
"""
从视频文件生成 Mesh 序列，使用与 SAM3DBodyProcess 相同的输入格式
注意: 此节点需要 SAM3DBody 节点的依赖
"""
@classmethod
def INPUT_TYPES(cls):
# 检查依赖是否可用
available, error_msg = check_sam3d_dependencies()
if not available:
print(f"警告: SAM3D依赖不可用，节点可能无法正常工作: {error_msg}")
print(f"Warning: ComfyUI-SAM3DBody dependency is unavailable; nodes may not function correctly: {error_msg}")
return {
"required": {
"model": ("SAM3D_MODEL", {
"tooltip": "SAM3D 模型配置"
"tooltip": "SAM3D Model"
}),
"image": ("IMAGE", {
"tooltip": "视频帧序列（批处理的图像）"
"tooltip": "Image Batch"
}),
"output_filename": ("STRING", {
"default": "mesh_sequence.bin",
"default": "mesh_sequence",
"multiline": False,
"placeholder": "输出文件名 (会自动添加时间戳)",
"tooltip": "输出二进制文件名，会保存在 Adv3DViewer_JK_tmp 目录"
"tooltip": "The output binary filename will be saved in the Adv3DViewer_JK_tmp directory of the output folder."
}),
},
"optional": {
@@ -411,30 +508,43 @@ class SAM3DMeshSequenceFromVideo_JK:
"min": 0.0,
"max": 1.0,
"step": 0.05,
"tooltip": "人体检测阈值"
"tooltip": "Human body detection threshold"
}),
"inference_type": (["full", "body", "hand"], {
"default": "full",
"tooltip": "推理类型"
"tooltip": "Inference type"
}),
"mask": ("MASK", {
"tooltip": "可选的分割掩码序列"
"tooltip": "Optional Mask"
}),
"coordinate_transform": (["none", "rotate_z_180", "rotate_y_180", "rotate_x_180", "rotate_z_90"], {
"default": "rotate_z_180",
"tooltip": "坐标变换类型。通常使用rotate_z_180修正朝向问题"
# "coordinate_transform": (["none", "rotate_z_180", "rotate_y_180", "rotate_x_180"], {
# "default": "rotate_z_180",
# "tooltip": "Coordinate transformation type. Rotate_z_180 is typically used to correct SAM3D Body orientation issues."
# }),
"smoothing_method": (["gaussian", "moving_average"], {
"default": "gaussian",
"tooltip": "Smoothing Method"
}),
"smoothing_sigma": ("FLOAT", {
"default": 3.0,
"min": 0.5,
"max": 10.0,
"step": 0.5,
"tooltip": "高斯平滑核宽度（越高越平滑）"
}),
"smoothing_method": (["gaussian", "moving_average"], {
"default": "gaussian",
"tooltip": "平滑算法"
"tooltip": "Gaussian smoothing kernel width (the higher the width, the smoother the surface)"
}),
# "align_to_reference_camera": ("BOOLEAN", {
# "default": True,
# "label_on": "Enabled",
# "label_off": "Disabled",
# "tooltip": "Align all models to the camera (fixed camera) of the reference frame to ensure perfect alignment with the video."
# }),
# "reference_frame": ("INT", {
# "default": 0,
# "min": 0,
# "max": 1000,
# "step": 1,
# "tooltip": "Reference frame index (used for camera alignment)"
# }),
}
}
@@ -449,7 +559,7 @@ class SAM3DMeshSequenceFromVideo_JK:
# 检查依赖是否可用
available, error_msg = check_sam3d_dependencies()
if not available:
print(f"警告: SAM3D依赖不可用，节点可能无法正常工作: {error_msg}")
print(f"[SAM3D Check] ComfyUI-SAM3DBody dependency is unavailable; nodes may not function correctly: {error_msg}")
import folder_paths
self.output_dir = folder_paths.get_output_directory()
@@ -459,8 +569,6 @@ class SAM3DMeshSequenceFromVideo_JK:
tmp_output_dir = Path(self.output_dir) / self.tmp_output_dir_name
tmp_output_dir.mkdir(parents=True, exist_ok=True)
print(f"[SAM3DMeshSequence] Output directory: {tmp_output_dir}")
def get_tmp_output_dir(self):
"""获取临时输出目录的 Path 对象"""
return Path(self.output_dir) / self.tmp_output_dir_name
@@ -496,27 +604,21 @@ class SAM3DMeshSequenceFromVideo_JK:
def generate_from_video_frames(self, model, image, output_filename,
bbox_threshold=0.8, inference_type="full", mask=None,
coordinate_transform="rotate_z_180",
smoothing_sigma=3.0, smoothing_method="gaussian"):
"""
从图像序列（视频帧）生成 Mesh 序列
"""
smoothing_sigma=3.0, smoothing_method="gaussian",
align_to_reference_camera=True, reference_frame=0):
import time
start_time_total = time.time()
# 检查依赖是否可用
available, error_msg = check_sam3d_dependencies()
if not available:
return (f"错误: SAM3D依赖不可用 - {error_msg}",)
return (f"[SAM3D Check] SAM3D dependency unavailable: {error_msg}",)
# 1. 检查输入图像
if image is None or len(image) == 0:
return (f"错误: 没有输入图像",)
return (f"[VideoFramesToMesh] Error: No input image",)
num_frames = len(image)
print(f"[VideoFramesToMesh] Input image sequence: {num_frames}frames")
print(f"[VideoFramesToMesh] Coordinate transformation: {coordinate_transform}")
print(f"[VideoFramesToMesh] Temporal smoothing: enabled, method: {smoothing_method}, sigma: {smoothing_sigma}")
# 2. 加载模型
model_start_time = time.time()
@@ -531,9 +633,7 @@ class SAM3DMeshSequenceFromVideo_JK:
try:
loaded = _load_sam3d_model_process(model_config)
except ImportError as e:
return (f"错误: 无法加载SAM3D模型 - {str(e)}",)
print(f"[VideoFramesToMesh] Model loaded, time: {time.time() - model_start_time:.2f}seconds")
return (f"[SAM3D Check] SAM3D dependency unavailable: {str(e)}",)
# 3. 获取面数据
from sam_3d_body import SAM3DBodyEstimator
@@ -557,7 +657,7 @@ class SAM3DMeshSequenceFromVideo_JK:
pass
if faces is None:
return (f"错误: 无法获取面数据",)
return (f"[VideoFramesToMesh] Error: Unable to retrieve face data",)
# 4. 准备输出路径
# 清理文件名
@@ -579,10 +679,8 @@ class SAM3DMeshSequenceFromVideo_JK:
tmp_output_dir = self.get_tmp_output_dir()
output_path = tmp_output_dir / unique_filename
print(f"[VideoFramesToMesh] Output file: {output_path}")
# 5. 处理图像序列 - 收集顶点信息
vertices_sequence = []
frames_data = [] # 存储每帧的完整数据
processed_frames = 0
failed_frames = 0
@@ -644,11 +742,13 @@ class SAM3DMeshSequenceFromVideo_JK:
print(f"[VideoFramesToMesh] Warning: Frame {frame_idx} no human detected")
failed_frames += 1
# 添加空白帧
num_verts = faces.shape[0]
blank_vertices = np.zeros((num_verts, 3))
vertices_sequence.append(blank_vertices)
# 添加空白帧数据
frames_data.append({
"vertices": None,
"cam_t": None,
"focal_length": None,
"success": False
})
processed_frames += 1
continue
@@ -657,11 +757,19 @@ class SAM3DMeshSequenceFromVideo_JK:
# 提取顶点
pred_vertices = output.get("pred_vertices")
# 提取相机参数
pred_cam_t = output.get("pred_cam_t")
focal_length = output.get("focal_length")
if pred_vertices is None:
print(f"[VideoFramesToMesh] Warning: Frame {frame_idx} no vertex output, using blank frame")
num_verts = faces.shape[0]
blank_vertices = np.zeros((num_verts, 3))
vertices_sequence.append(blank_vertices)
frames_data.append({
"vertices": None,
"cam_t": None,
"focal_length": None,
"success": False
})
else:
# 转换为 numpy 数组
if torch.is_tensor(pred_vertices):
@@ -669,7 +777,24 @@ class SAM3DMeshSequenceFromVideo_JK:
else:
vertices = pred_vertices
vertices_sequence.append(vertices)
# 处理相机参数
if torch.is_tensor(pred_cam_t):
cam_t = pred_cam_t.detach().cpu().numpy()
else:
cam_t = pred_cam_t
if focal_length is not None:
if hasattr(focal_length, 'item'):
focal_length = focal_length.item()
focal_length = float(focal_length)
# 保存帧数据
frames_data.append({
"vertices": vertices,
"cam_t": cam_t,
"focal_length": focal_length,
"success": True
})
processed_frames += 1
@@ -677,10 +802,12 @@ class SAM3DMeshSequenceFromVideo_JK:
print(f"[VideoFramesToMesh] Error: Processing frame {frame_idx} failed - {e}")
failed_frames += 1
# 添加空白帧
num_verts = faces.shape[0]
blank_vertices = np.zeros((num_verts, 3))
vertices_sequence.append(blank_vertices)
frames_data.append({
"vertices": None,
"cam_t": None,
"focal_length": None,
"success": False
})
processed_frames += 1
@@ -695,18 +822,47 @@ class SAM3DMeshSequenceFromVideo_JK:
if processed_frames == 0:
return (f"错误: 没有成功处理任何帧",)
# 6. 转换为 numpy 数组
vertices_sequence_np = np.stack(vertices_sequence, axis=0)
# 6. 根据参数决定是否应用模型对齐
if align_to_reference_camera:
print(f"[VideoFramesToMesh] Aligning models to fixed camera (reference frame: {reference_frame})...")
try:
aligned_vertices, valid_frames, ref_cam_params = align_models_to_fixed_camera(
frames_data,
faces,
reference_frame=reference_frame
)
print(f"[Alignment] Reference camera parameters:")
print(f" cam_t: {ref_cam_params['cam_t'].flatten()}")
print(f" focal_length: {ref_cam_params['focal_length']}")
except Exception as e:
print(f"[Alignment] Error during alignment: {e}")
print(f"[Alignment] Falling back to original vertices")
# 回退：使用原始顶点（不应用对齐）
aligned_vertices = np.stack([d["vertices"] if d["success"] else np.zeros((faces.shape[0], 3))
for d in frames_data], axis=0)
valid_frames = [i for i, d in enumerate(frames_data) if d["success"]]
else:
print(f"[VideoFramesToMesh] Skipping model alignment (align_to_reference_camera=False)")
# 使用原始顶点
aligned_vertices = np.stack([d["vertices"] if d["success"] else np.zeros((faces.shape[0], 3))
for d in frames_data], axis=0)
valid_frames = [i for i, d in enumerate(frames_data) if d["success"]]
# 7. 总是应用时间平滑（如果有多于1帧）
if processed_frames > 1:
if len(valid_frames) > 1:
print(f"[VideoFramesToMesh] Applying temporal smoothing (method: {smoothing_method}, sigma: {smoothing_sigma})")
vertices_sequence_np = self.smooth_sequence(
vertices_sequence_np,
# 只对有效帧应用平滑
valid_vertices = aligned_vertices[valid_frames]
smoothed_vertices = self.smooth_sequence(
valid_vertices,
sigma=smoothing_sigma,
method=smoothing_method
)
aligned_vertices[valid_frames] = smoothed_vertices
print(f"[VideoFramesToMesh] Temporal smoothing completed")
@@ -715,19 +871,20 @@ class SAM3DMeshSequenceFromVideo_JK:
print(f"[CoordinateTransform] Applying coordinate transformation: {coordinate_transform}")
# 变换顶点序列
vertices_sequence_np = apply_coordinate_transform(vertices_sequence_np, coordinate_transform)
aligned_vertices = apply_coordinate_transform(aligned_vertices, coordinate_transform)
# 9. 总是保存为 SMPL 兼容格式
MeshSequenceBinaryFormat.save_smpl_compatible(
vertices_sequence=vertices_sequence_np,
vertices_sequence=aligned_vertices,
faces=faces,
output_path=output_path,
fps=DEFAULT_FPS,
coordinate_transform="none" # 已经在前面应用了变换
)
print(f"[VideoFramesToMesh] SMPL compatible format generation completed! Total time: {time.time() - start_time_total:.2f}seconds")
print(f"[VideoFramesToMesh] Successfully processed: {processed_frames - failed_frames}frames, Failed: {failed_frames}frames")
print(f"[VideoFramesToMesh] Model alignment and generation completed!")
print(f"[VideoFramesToMesh] Successfully processed: {processed_frames - failed_frames} frames, Failed: {failed_frames} frames")
print(f"[VideoFramesToMesh] Model alignment: enabled (reference frame: {ref_cam_params.get('reference_frame', 0)})")
print(f"[VideoFramesToMesh] Coordinate transformation: {coordinate_transform}")
print(f"[VideoFramesToMesh] Temporal smoothing: enabled ({smoothing_method}, sigma={smoothing_sigma})")
print(f"[VideoFramesToMesh] Output file: {output_path}")
pyproject.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "comfyui-jakeupgrade"
description = "ComfyUI workflow customization by Jake."
version = "2.5.3"
version = "2.5.4"
license = {file = "LICENSE"}
dependencies = ["opencv-python", "piexif", "torch", "torchvision", "numpy", "pillow", "simpleeval", "PyYAML", "toml"]