146 lines
8.2 KiB
Markdown
146 lines
8.2 KiB
Markdown
# Documentation
|
|
- Class name: MotionctrlSample
|
|
- Category: motionctrl
|
|
- Output node: False
|
|
- Repo Ref: https://github.com/chaojie/ComfyUI-MotionCtrl.git
|
|
|
|
MotionctrlSample 类旨在通过将运动控制融入渲染过程来促进动态视觉内容的生成。它利用先进算法的力量解释用户定义的相机和轨迹输入，使得能够创造出带有逼真动画的复杂运动序列。该节点抽象了运动模拟的复杂性，允许用户专注于创造性表达而不是底层技术细节。
|
|
|
|
# Input types
|
|
## Required
|
|
- prompt
|
|
- 提示参数作为MotionctrlSample节点的创意输入，指导生成内容的整体主题和风格。它对于设置运动模拟发生的上下文至关重要，直接影响渲染过程中产生的叙述和视觉元素。
|
|
- Comfy dtype: STRING
|
|
- Python dtype: str
|
|
- camera
|
|
- 相机参数定义了捕捉运动序列的视角和视点。它在塑造渲染场景中的时空关系和深度感知方面起着关键作用，从而显著影响最终的视觉输出。
|
|
- Comfy dtype: STRING
|
|
- Python dtype: str
|
|
- traj
|
|
- 轨迹参数概述了场景中物体的路径和运动，成为运动模拟的基础。它对于指导动画的动态性和元素的相互作用至关重要，确保了运动表现的连贯性和流畅性。
|
|
- Comfy dtype: STRING
|
|
- Python dtype: str
|
|
- frame_length
|
|
- 帧长度参数指定了动画序列的持续时间，直接影响运动轨迹的详细程度和复杂性。较长的帧长度允许更复杂的运动模式和更平滑的过渡，而较短的帧长度可能导致更简单或更突然的运动。
|
|
- Comfy dtype: INT
|
|
- Python dtype: int
|
|
- steps
|
|
- 步数参数决定了运动模拟过程中生成的中间帧数，这影响动画的平滑度和质量。步数越高，运动轨迹越精细，而步数较低可能导致动画细节不足或不连贯。
|
|
- Comfy dtype: INT
|
|
- Python dtype: int
|
|
- seed
|
|
- 种子参数用于初始化随机数生成器，确保运动模拟结果是可复现和一致的。这对于调试和比较不同模拟运行是重要的方面。
|
|
- Comfy dtype: INT
|
|
- Python dtype: int
|
|
|
|
# Output types
|
|
- output
|
|
- MotionctrlSample 节点的输出是一个动态视觉表示，以图像或图像序列的形式，封装了运动模拟的结果。它反映了输入参数的综合效应，展示了生成的运动内容的创造性和复杂性。
|
|
- Comfy dtype: IMAGE
|
|
- Python dtype: torch.Tensor
|
|
|
|
# Usage tips
|
|
- Infra type: GPU
|
|
|
|
# Source code
|
|
```
|
|
class MotionctrlSample:
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {'required': {'prompt': ('STRING', {'multiline': True, 'default': 'a rose swaying in the wind'}), 'camera': ('STRING', {'multiline': True, 'default': '[[1,0,0,0,0,1,0,0,0,0,1,0.2]]'}), 'traj': ('STRING', {'multiline': True, 'default': '[[117, 102]]'}), 'frame_length': ('INT', {'default': 16}), 'steps': ('INT', {'default': 50}), 'seed': ('INT', {'default': 1234})}, 'optional': {'traj_tool': ('STRING', {'multiline': False, 'default': 'https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html'}), 'draw_traj_dot': ('BOOLEAN', {'default': False}), 'draw_camera_dot': ('BOOLEAN', {'default': False}), 'ckpt_name': (['motionctrl.pth'], {'default': 'motionctrl.pth'})}}
|
|
RETURN_TYPES = ('IMAGE',)
|
|
FUNCTION = 'run_inference'
|
|
CATEGORY = 'motionctrl'
|
|
|
|
def run_inference(self, prompt, camera, traj, frame_length, steps, seed, traj_tool='https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html', draw_traj_dot=False, draw_camera_dot=False, ckpt_name='motionctrl.pth'):
|
|
gpu_num = 1
|
|
gpu_no = 0
|
|
ckpt_path = folder_paths.get_full_path('checkpoints', ckpt_name)
|
|
comfy_path = os.path.dirname(folder_paths.__file__)
|
|
config_path = os.path.join(comfy_path, 'custom_nodes/ComfyUI-MotionCtrl/configs/inference/config_both.yaml')
|
|
args = {'savedir': f'./output/both_seed20230211', 'ckpt_path': f'{ckpt_path}', 'adapter_ckpt': None, 'base': f'{config_path}', 'condtype': 'both', 'prompt_dir': None, 'n_samples': 1, 'ddim_steps': 50, 'ddim_eta': 1.0, 'bs': 1, 'height': 256, 'width': 256, 'unconditional_guidance_scale': 1.0, 'unconditional_guidance_scale_temporal': None, 'seed': 1234, 'cond_T': 800, 'save_imgs': True, 'cond_dir': './custom_nodes/ComfyUI-MotionCtrl/examples/'}
|
|
prompts = prompt
|
|
RT = process_camera(camera, frame_length).reshape(-1, 12)
|
|
RT_list = process_camera_list(camera, frame_length)
|
|
traj_flow = process_traj(traj, frame_length).transpose(3, 0, 1, 2)
|
|
print(prompts)
|
|
print(RT.shape)
|
|
print(traj_flow.shape)
|
|
args['savedir'] = f"./output/{args['condtype']}_seed{args['seed']}"
|
|
config = OmegaConf.load(args['base'])
|
|
OmegaConf.update(config, 'model.params.unet_config.params.temporal_length', frame_length)
|
|
model_config = config.pop('model', OmegaConf.create())
|
|
model = instantiate_from_config(model_config)
|
|
model = model.cuda(gpu_no)
|
|
assert os.path.exists(args['ckpt_path']), f"Error: checkpoint {args['ckpt_path']} Not Found!"
|
|
print(f"Loading checkpoint from {args['ckpt_path']}")
|
|
model = load_model_checkpoint(model, args['ckpt_path'], args['adapter_ckpt'])
|
|
model.eval()
|
|
assert args['height'] % 16 == 0 and args['width'] % 16 == 0, 'Error: image size [h,w] should be multiples of 16!'
|
|
(h, w) = (args['height'] // 8, args['width'] // 8)
|
|
channels = model.channels
|
|
frames = model.temporal_length
|
|
noise_shape = [args['bs'], channels, frames, h, w]
|
|
savedir = os.path.join(args['savedir'], 'samples')
|
|
os.makedirs(savedir, exist_ok=True)
|
|
unconditional_guidance_scale = 7.5
|
|
unconditional_guidance_scale_temporal = None
|
|
n_samples = 1
|
|
ddim_steps = steps
|
|
ddim_eta = 1.0
|
|
cond_T = 800
|
|
if n_samples < 1:
|
|
n_samples = 1
|
|
if n_samples > 4:
|
|
n_samples = 4
|
|
seed_everything(seed)
|
|
camera_poses = RT
|
|
trajs = traj_flow
|
|
camera_poses = torch.tensor(camera_poses).float()
|
|
trajs = torch.tensor(trajs).float()
|
|
camera_poses = camera_poses.unsqueeze(0)
|
|
trajs = trajs.unsqueeze(0)
|
|
if torch.cuda.is_available():
|
|
camera_poses = camera_poses.cuda()
|
|
trajs = trajs.cuda()
|
|
ddim_sampler = DDIMSampler(model)
|
|
batch_size = noise_shape[0]
|
|
prompts = prompt
|
|
if isinstance(prompts, str):
|
|
prompts = [prompts]
|
|
for i in range(len(prompts)):
|
|
prompts[i] = f'{prompts[i]}, {post_prompt}'
|
|
cond = model.get_learned_conditioning(prompts)
|
|
if camera_poses is not None:
|
|
RT = camera_poses[..., None]
|
|
else:
|
|
RT = None
|
|
traj_features = None
|
|
if trajs is not None:
|
|
traj_features = model.get_traj_features(trajs)
|
|
else:
|
|
traj_features = None
|
|
uc = None
|
|
if unconditional_guidance_scale != 1.0:
|
|
prompts = batch_size * [DEFAULT_NEGATIVE_PROMPT]
|
|
uc = model.get_learned_conditioning(prompts)
|
|
if traj_features is not None:
|
|
un_motion = model.get_traj_features(torch.zeros_like(trajs))
|
|
else:
|
|
un_motion = None
|
|
uc = {'features_adapter': un_motion, 'uc': uc}
|
|
else:
|
|
uc = None
|
|
batch_images = []
|
|
batch_variants = []
|
|
for _ in range(n_samples):
|
|
if ddim_sampler is not None:
|
|
(samples, _) = ddim_sampler.sample(S=ddim_steps, conditioning=cond, batch_size=noise_shape[0], shape=noise_shape[1:], verbose=False, unconditional_guidance_scale=unconditional_guidance_scale, unconditional_conditioning=uc, eta=ddim_eta, temporal_length=noise_shape[2], conditional_guidance_scale_temporal=unconditional_guidance_scale_temporal, features_adapter=traj_features, pose_emb=RT, cond_T=cond_T)
|
|
batch_images = model.decode_first_stage(samples)
|
|
batch_variants.append(batch_images)
|
|
batch_variants = torch.stack(batch_variants, dim=1)
|
|
batch_variants = batch_variants[0]
|
|
ret = save_results(batch_variants, fps=10, traj=traj, draw_traj_dot=draw_traj_dot, cameras=RT_list, draw_camera_dot=draw_camera_dot)
|
|
return ret
|
|
``` |