Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 fles
|
919db1c3c7 | ||
|
fles
|
6bba33e306 | ||
|
fles@qq.com
|
f0873a89d4 | ||
|
fles
|
8e918def58 | ||
|
fles
|
b0f94320c1 | ||
|
fles
|
cd9142d8cb | ||
|
fles
|
57a8700b1c |
+2
-1
@@ -3,4 +3,5 @@
|
||||
|
||||
*.pyc
|
||||
gradio_temp
|
||||
*.pth
|
||||
*.pth
|
||||
/turbo/dist/index.html
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
## Nodes
|
||||
|
||||
Four nodes `Motionctrl Sample` & `Load Motion Camera Preset` & `Load Motion Traj Preset` & `Select Image Indices`
|
||||
Four nodes `Load Motionctrl Checkpoint` & `Motionctrl Cond` & `Motionctrl Sample Simple` & `Load Motion Camera Preset` & `Load Motion Traj Preset` & `Select Image Indices` &`Motionctrl Sample`
|
||||
|
||||
## Tools
|
||||
|
||||
@@ -30,7 +30,7 @@ base workflow
|
||||
|
||||
<img src="assets/base_wf.png" raw=true>
|
||||
|
||||
https://github.com/chaojie/ComfyUI-MotionCtrl/blob/main/workflow_threenodes.json
|
||||
https://github.com/chaojie/ComfyUI-MotionCtrl/blob/main/workflow_motionctrl_base.json
|
||||
|
||||
<video controls autoplay="true">
|
||||
<source
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 186 KiB After Width: | Height: | Size: 304 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 119 KiB After Width: | Height: | Size: 154 KiB |
@@ -70,7 +70,7 @@ def process_traj(points_str,frame_length):
|
||||
|
||||
return optical_flow
|
||||
|
||||
def save_results(video, fps=10,traj="[]",draw_traj_dot=False,cameras=[],draw_camera_dot=False):
|
||||
def save_results(video, fps=10,traj="[]",draw_traj_dot=False,cameras=[],draw_camera_dot=False,context_overlap=0):
|
||||
|
||||
# b,c,t,h,w
|
||||
video = video.detach().cpu()
|
||||
@@ -117,7 +117,7 @@ def save_results(video, fps=10,traj="[]",draw_traj_dot=False,cameras=[],draw_cam
|
||||
#writer.append_data(img)
|
||||
|
||||
#writer.close()
|
||||
return torch.cat(tuple(outframes), dim=0).unsqueeze(0)
|
||||
return torch.cat(tuple(outframes[context_overlap:]), dim=0).unsqueeze(0)
|
||||
|
||||
MOTION_CAMERA_OPTIONS = ["U", "D", "L", "R", "O", "O_0.2x", "O_0.4x", "O_1.0x", "O_2.0x", "O_0.2x", "O_0.2x", "Round-RI", "Round-RI_90", "Round-RI-120", "Round-ZoomIn", "SPIN-ACW-60", "SPIN-CW-60", "I", "I_0.2x", "I_0.4x", "I_1.0x", "I_2.0x", "1424acd0007d40b5", "d971457c81bca597", "018f7907401f2fef", "088b93f15ca8745d", "b133a504fc90a2d1"]
|
||||
|
||||
@@ -184,14 +184,311 @@ class LoadMotionTrajPreset:
|
||||
points = read_points(f'{comfy_path}/custom_nodes/ComfyUI-MotionCtrl/examples/trajectories/{motion_traj}.txt',frame_length)
|
||||
return (json.dumps(points),)
|
||||
|
||||
MODE = ["control camera poses", "control object trajectory", "control both camera and object motion"]
|
||||
class MotionctrlLoader:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"ckpt_name": (folder_paths.get_filename_list("checkpoints"), {"default": "motionctrl.pth"}),
|
||||
"frame_length": ("INT", {"default": 16}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MOTIONCTRL", "EMBEDDER", "VAE", "SAMPLER",)
|
||||
RETURN_NAMES = ("model","clip","vae","ddim_sampler",)
|
||||
FUNCTION = "load_checkpoint"
|
||||
CATEGORY = "motionctrl"
|
||||
|
||||
def load_checkpoint(self, ckpt_name, frame_length):
|
||||
gpu_num=1
|
||||
gpu_no=0
|
||||
ckpt_path = folder_paths.get_full_path("checkpoints", ckpt_name)
|
||||
comfy_path = os.path.dirname(folder_paths.__file__)
|
||||
config_path = os.path.join(comfy_path, 'custom_nodes/ComfyUI-MotionCtrl/configs/inference/config_both.yaml')
|
||||
args={"ckpt_path":f"{ckpt_path}","adapter_ckpt":None,"base":f"{config_path}","condtype":"both","prompt_dir":None,"n_samples":1,"ddim_steps":50,"ddim_eta":1.0,"bs":1,"height":256,"width":256,"unconditional_guidance_scale":1.0,"unconditional_guidance_scale_temporal":None,"seed":1234,"cond_T":800}
|
||||
|
||||
config = OmegaConf.load(args["base"])
|
||||
OmegaConf.update(config, "model.params.unet_config.params.temporal_length", frame_length)
|
||||
model_config = config.pop("model", OmegaConf.create())
|
||||
model = instantiate_from_config(model_config)
|
||||
model = model.cuda(gpu_no)
|
||||
assert os.path.exists(args["ckpt_path"]), f'Error: checkpoint {args["ckpt_path"]} Not Found!'
|
||||
print(f'Loading checkpoint from {args["ckpt_path"]}')
|
||||
model = load_model_checkpoint(model, args["ckpt_path"], args["adapter_ckpt"])
|
||||
model.eval()
|
||||
|
||||
ddim_sampler = DDIMSampler(model)
|
||||
|
||||
return (model,model.cond_stage_model,model.first_stage_model,ddim_sampler,)
|
||||
|
||||
|
||||
class MotionctrlCond:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MOTIONCTRL",),
|
||||
"prompt": ("STRING", {"multiline": True, "default":"a rose swaying in the wind"}),
|
||||
"camera": ("STRING", {"multiline": True, "default":"[[1,0,0,0,0,1,0,0,0,0,1,0.2]]"}),
|
||||
"traj": ("STRING", {"multiline": True, "default":"[[117, 102]]"}),
|
||||
"infer_mode": (MODE, {"default":"control both camera and object motion"}),
|
||||
"context_overlap": ("INT", {"default": 0, "min": 0, "max": 32}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("CONDITIONING", "CONDITIONING","TRAJ_LIST","RT_LIST","TRAJ_FEATURES","RT","NOISE_SHAPE","INT")
|
||||
RETURN_NAMES = ("positive", "negative","traj_list","rt_list","traj","rt","noise_shape","context_overlap")
|
||||
FUNCTION = "load_cond"
|
||||
CATEGORY = "motionctrl"
|
||||
|
||||
def load_cond(self, model, prompt, camera, traj,infer_mode,context_overlap):
|
||||
comfy_path = os.path.dirname(folder_paths.__file__)
|
||||
camera_align_file = os.path.join(comfy_path, 'custom_nodes/ComfyUI-MotionCtrl/camera.json')
|
||||
traj_align_file = os.path.join(comfy_path, 'custom_nodes/ComfyUI-MotionCtrl/traj.json')
|
||||
frame_length=model.temporal_length
|
||||
|
||||
camera_align=json.loads(camera)
|
||||
for i in range(frame_length):
|
||||
if len(camera_align)<=i:
|
||||
camera_align.append(camera_align[len(camera_align)-1])
|
||||
camera=json.dumps(camera_align)
|
||||
traj_align=json.loads(traj)
|
||||
for i in range(frame_length):
|
||||
if len(traj_align)<=i:
|
||||
traj_align.append(traj_align[len(traj_align)-1])
|
||||
traj=json.dumps(traj_align)
|
||||
|
||||
if context_overlap>0:
|
||||
if os.path.exists(camera_align_file):
|
||||
with open(camera_align_file, 'r') as file:
|
||||
pre_camera_align=json.load(file)
|
||||
camera_align=pre_camera_align[:context_overlap]+camera_align[:-context_overlap]
|
||||
|
||||
if os.path.exists(traj_align_file):
|
||||
with open(traj_align_file, 'r') as file:
|
||||
pre_traj_align=json.load(file)
|
||||
traj_align=pre_traj_align[:context_overlap]+traj_align[:-context_overlap]
|
||||
|
||||
with open(camera_align_file, 'w') as file:
|
||||
json.dump(camera_align, file)
|
||||
|
||||
with open(traj_align_file, 'w') as file:
|
||||
json.dump(traj_align, file)
|
||||
|
||||
prompts = prompt
|
||||
RT = process_camera(camera,frame_length).reshape(-1,12)
|
||||
RT_list = process_camera_list(camera,frame_length)
|
||||
traj_flow = process_traj(traj,frame_length).transpose(3,0,1,2)
|
||||
print(prompts)
|
||||
print(RT.shape)
|
||||
print(traj_flow.shape)
|
||||
|
||||
height=256
|
||||
width=256
|
||||
|
||||
## run over data
|
||||
assert (height % 16 == 0) and (width % 16 == 0), "Error: image size [h,w] should be multiples of 16!"
|
||||
|
||||
## latent noise shape
|
||||
h, w = height // 8, width // 8
|
||||
channels = model.channels
|
||||
frames = model.temporal_length
|
||||
#frames = frame_length
|
||||
noise_shape = [1, channels, frames, h, w]
|
||||
|
||||
if infer_mode == MODE[0]:
|
||||
camera_poses = RT
|
||||
camera_poses = torch.tensor(camera_poses).float()
|
||||
camera_poses = camera_poses.unsqueeze(0)
|
||||
trajs = None
|
||||
if torch.cuda.is_available():
|
||||
camera_poses = camera_poses.cuda()
|
||||
elif infer_mode == MODE[1]:
|
||||
trajs = traj_flow
|
||||
trajs = torch.tensor(trajs).float()
|
||||
trajs = trajs.unsqueeze(0)
|
||||
camera_poses = None
|
||||
if torch.cuda.is_available():
|
||||
trajs = trajs.cuda()
|
||||
else:
|
||||
camera_poses = RT
|
||||
trajs = traj_flow
|
||||
camera_poses = torch.tensor(camera_poses).float()
|
||||
trajs = torch.tensor(trajs).float()
|
||||
camera_poses = camera_poses.unsqueeze(0)
|
||||
trajs = trajs.unsqueeze(0)
|
||||
if torch.cuda.is_available():
|
||||
camera_poses = camera_poses.cuda()
|
||||
trajs = trajs.cuda()
|
||||
|
||||
batch_size = noise_shape[0]
|
||||
prompts=prompt
|
||||
## get condition embeddings (support single prompt only)
|
||||
if isinstance(prompts, str):
|
||||
prompts = [prompts]
|
||||
|
||||
for i in range(len(prompts)):
|
||||
prompts[i] = f'{prompts[i]}, {post_prompt}'
|
||||
|
||||
cond = model.get_learned_conditioning(prompts)
|
||||
if camera_poses is not None:
|
||||
RT = camera_poses[..., None]
|
||||
else:
|
||||
RT = None
|
||||
|
||||
traj_features = None
|
||||
if trajs is not None:
|
||||
traj_features = model.get_traj_features(trajs)
|
||||
else:
|
||||
traj_features = None
|
||||
|
||||
uc = None
|
||||
prompts = batch_size * [DEFAULT_NEGATIVE_PROMPT]
|
||||
uc = model.get_learned_conditioning(prompts)
|
||||
if traj_features is not None:
|
||||
un_motion = model.get_traj_features(torch.zeros_like(trajs))
|
||||
else:
|
||||
un_motion = None
|
||||
uc = {"features_adapter": un_motion, "uc": uc}
|
||||
|
||||
return (cond,uc,traj,RT_list,traj_features,RT,noise_shape,context_overlap)
|
||||
|
||||
|
||||
|
||||
class MotionctrlSampleSimple:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"model": ("MOTIONCTRL",),
|
||||
"clip": ("EMBEDDER",),
|
||||
"vae": ("VAE",),
|
||||
"ddim_sampler": ("SAMPLER",),
|
||||
"positive": ("CONDITIONING",),
|
||||
"negative": ("CONDITIONING",),
|
||||
"traj_list": ("TRAJ_LIST",),
|
||||
"rt_list": ("RT_LIST",),
|
||||
"traj": ("TRAJ_FEATURES",),
|
||||
"rt": ("RT",),
|
||||
"steps": ("INT", {"default": 50}),
|
||||
"seed": ("INT", {"default": 1234}),
|
||||
"noise_shape":("NOISE_SHAPE",),
|
||||
"context_overlap": ("INT", {"default": 0, "min": 0, "max": 32}),
|
||||
},
|
||||
"optional": {
|
||||
"traj_tool": ("STRING",{"multiline": False, "default": "https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html"}),
|
||||
"draw_traj_dot": ("BOOLEAN", {"default": False}),#, "label_on": "draw", "label_off": "not draw"
|
||||
"draw_camera_dot": ("BOOLEAN", {"default": False}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "run_inference"
|
||||
CATEGORY = "motionctrl"
|
||||
|
||||
def run_inference(self,model,clip,vae,ddim_sampler,positive, negative,traj_list,rt_list,traj,rt,steps,seed,noise_shape,context_overlap,traj_tool="https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html",draw_traj_dot=False,draw_camera_dot=False):
|
||||
frame_length=model.temporal_length
|
||||
device = model.betas.device
|
||||
print(f'frame_length{frame_length}')
|
||||
#noise_shape = [1, 4, 16, 32, 32]
|
||||
unconditional_guidance_scale = 7.5
|
||||
unconditional_guidance_scale_temporal = None
|
||||
n_samples = 1
|
||||
ddim_steps= steps
|
||||
ddim_eta=1.0
|
||||
cond_T=800
|
||||
#seed = args["seed"]
|
||||
|
||||
if n_samples < 1:
|
||||
n_samples = 1
|
||||
if n_samples > 4:
|
||||
n_samples = 4
|
||||
|
||||
seed_everything(seed)
|
||||
|
||||
batch_images=[]
|
||||
batch_variants = []
|
||||
intermediates = {}
|
||||
|
||||
x0=None
|
||||
x_T=None
|
||||
pre_x0=None
|
||||
pre_x_T=None
|
||||
|
||||
comfy_path = os.path.dirname(folder_paths.__file__)
|
||||
pred_x0_path = os.path.join(comfy_path, 'custom_nodes/ComfyUI-MotionCtrl/pred_x0.pt')
|
||||
x_inter_path = os.path.join(comfy_path, 'custom_nodes/ComfyUI-MotionCtrl/x_inter.pt')
|
||||
randt=torch.randn([noise_shape[0],noise_shape[1],frame_length-context_overlap,noise_shape[3],noise_shape[4]], device=device)
|
||||
randt_np=randt.detach().cpu().numpy()
|
||||
|
||||
if context_overlap>0:
|
||||
if os.path.exists(pred_x0_path):
|
||||
pre_x0=torch.load(pred_x0_path)
|
||||
pre_x0_np=pre_x0[-1].detach().cpu().numpy()
|
||||
pre_x0_np_overlap = np.concatenate((pre_x0_np[:,:,-context_overlap:], randt_np), axis=2)
|
||||
x0=torch.tensor(pre_x0_np_overlap, device=device)
|
||||
if os.path.exists(x_inter_path):
|
||||
pre_x_T=torch.load(x_inter_path)
|
||||
pre_x_T_np=pre_x_T[-1].detach().cpu().numpy()
|
||||
pre_x_T_np_overlap = np.concatenate((pre_x_T_np[:,:,-context_overlap:], randt_np), axis=2)
|
||||
x_T=torch.tensor(pre_x_T_np_overlap, device=device)
|
||||
|
||||
for _ in range(n_samples):
|
||||
if ddim_sampler is not None:
|
||||
samples, intermediates = ddim_sampler.sample(S=ddim_steps,
|
||||
conditioning=positive,
|
||||
batch_size=noise_shape[0],
|
||||
shape=noise_shape[1:],
|
||||
verbose=False,
|
||||
unconditional_guidance_scale=unconditional_guidance_scale,
|
||||
unconditional_conditioning=negative,
|
||||
eta=ddim_eta,
|
||||
temporal_length=noise_shape[2],
|
||||
conditional_guidance_scale_temporal=unconditional_guidance_scale_temporal,
|
||||
features_adapter=traj,
|
||||
pose_emb=rt,
|
||||
cond_T=cond_T,
|
||||
x0=x0,
|
||||
x_T=x_T
|
||||
)
|
||||
#print(f'{samples}')
|
||||
## reconstruct from latent to pixel space
|
||||
batch_images = model.decode_first_stage(samples)
|
||||
batch_variants.append(batch_images)
|
||||
'''
|
||||
batch_images = model.decode_first_stage(intermediates['pred_x0'][0])
|
||||
batch_variants.append(batch_images)
|
||||
batch_images = model.decode_first_stage(intermediates['pred_x0'][1])
|
||||
batch_variants.append(batch_images)
|
||||
batch_images = model.decode_first_stage(intermediates['pred_x0'][2])
|
||||
batch_variants.append(batch_images)
|
||||
batch_images = model.decode_first_stage(intermediates['x_inter'][0])
|
||||
batch_variants.append(batch_images)
|
||||
batch_images = model.decode_first_stage(intermediates['x_inter'][1])
|
||||
batch_variants.append(batch_images)
|
||||
batch_images = model.decode_first_stage(intermediates['x_inter'][2])
|
||||
batch_variants.append(batch_images)
|
||||
'''
|
||||
## variants, batch, c, t, h, w
|
||||
batch_variants = torch.stack(batch_variants, dim=1)
|
||||
batch_variants = batch_variants[0]
|
||||
|
||||
torch.save(intermediates['x_inter'], x_inter_path)
|
||||
torch.save(intermediates['pred_x0'], pred_x0_path)
|
||||
ret = save_results(batch_variants, fps=10,traj=traj_list,draw_traj_dot=draw_traj_dot,cameras=rt_list,draw_camera_dot=draw_camera_dot,context_overlap=context_overlap)
|
||||
#print(ret)
|
||||
return ret
|
||||
|
||||
|
||||
class MotionctrlSample:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"prompt": ("STRING", {"multiline": True, "default":"a rose swaying in the wind"}),
|
||||
"camera": ("STRING", {"multiline": True, "default":"[[1,0,0,0,0,1,0,0,0,0,1,0.2],[1,0,0,0,0,1,0,0,0,0,1,0.28750000000000003],[1,0,0,0,0,1,0,0,0,0,1,0.37500000000000006],[1,0,0,0,0,1,0,0,0,0,1,0.4625000000000001],[1,0,0,0,0,1,0,0,0,0,1,0.55],[1,0,0,0,0,1,0,0,0,0,1,0.6375000000000002],[1,0,0,0,0,1,0,0,0,0,1,0.7250000000000001],[1,0,0,0,0,1,0,0,0,0,1,0.8125000000000002],[1,0,0,0,0,1,0,0,0,0,1,0.9000000000000001],[1,0,0,0,0,1,0,0,0,0,1,0.9875000000000003],[1,0,0,0,0,1,0,0,0,0,1,1.0750000000000002],[1,0,0,0,0,1,0,0,0,0,1,1.1625000000000003],[1,0,0,0,0,1,0,0,0,0,1,1.2500000000000002],[1,0,0,0,0,1,0,0,0,0,1,1.3375000000000001],[1,0,0,0,0,1,0,0,0,0,1,1.4250000000000003],[1,0,0,0,0,1,0,0,0,0,1,1.5125000000000004]]"}),
|
||||
"traj": ("STRING", {"multiline": True, "default":"[[117, 102],[114, 102],[109, 102],[106, 102],[105, 102],[102, 102],[99, 102],[97, 102],[96, 102],[95, 102],[93, 102],[89, 102],[85, 103],[82, 103],[81, 103],[80, 103],[79, 103],[78, 103],[76, 103],[74, 104],[73, 104],[72, 104],[71, 104],[70, 105],[69, 105],[68, 105],[67, 105],[66, 106],[64, 107],[63, 108],[62, 108],[61, 108],[61, 109],[60, 109],[59, 109],[58, 109],[57, 110],[56, 110],[55, 111],[54, 111],[53, 111],[52, 111],[52, 112],[51, 112],[50, 112],[50, 113],[49, 113],[48, 113],[46, 114],[46, 115],[45, 115],[45, 116],[44, 116],[43, 117],[42, 117],[41, 117],[41, 118],[40, 118],[41, 118],[41, 119],[42, 119],[43, 119],[44, 119],[46, 119],[47, 119],[48, 119],[49, 119],[50, 119],[51, 119],[52, 119],[53, 119],[54, 119],[55, 119],[56, 118],[58, 118],[59, 118],[61, 118],[63, 118],[64, 117],[67, 117],[70, 117],[71, 117],[73, 117],[75, 116],[76, 116],[77, 116],[80, 116],[82, 116],[83, 116],[84, 116],[85, 116],[88, 116],[91, 116],[94, 116],[97, 116],[98, 116],[100, 116],[101, 117],[102, 117],[104, 117],[105, 117],[106, 117],[107, 117],[108, 117],[109, 117],[110, 117],[111, 117],[115, 117],[119, 117],[123, 117],[124, 117],[128, 117],[129, 117],[132, 117],[134, 117],[135, 117],[136, 117],[138, 117],[139, 117],[140, 117],[141, 117],[142, 116],[145, 116],[146, 116],[148, 116],[149, 116],[151, 115],[152, 115],[153, 115],[154, 115],[155, 114],[156, 114],[157, 114],[158, 114],[159, 114],[162, 114],[163, 113],[164, 113],[165, 113],[166, 113],[167, 113],[168, 113],[169, 113],[170, 113],[171, 113],[172, 113],[173, 113],[174, 113],[175, 113],[178, 113],[181, 113],[182, 113],[183, 113],[184, 113],[185, 113],[187, 113],[188, 113],[189, 113],[191, 113],[192, 113],[193, 113],[194, 113],[195, 113],[196, 113],[197, 113],[198, 113],[199, 113],[200, 113],[201, 113],[202, 113],[203, 113],[202, 113],[201, 113],[200, 113],[198, 113],[197, 113],[196, 113],[195, 112],[194, 112],[193, 112],[192, 112],[191, 111],[190, 111],[189, 111],[188, 110],[187, 110],[186, 110],[185, 110],[184, 110],[183, 110],[182, 110],[181, 110],[180, 110],[179, 110],[178, 110],[177, 110],[175, 110],[173, 110],[172, 110],[171, 110],[170, 110],[168, 110],[167, 110],[165, 110],[164, 110],[163, 110],[161, 111],[159, 111],[155, 111],[153, 111],[151, 111],[151, 112],[150, 112],[149, 112],[148, 112],[147, 112],[145, 112],[143, 113],[142, 113],[140, 113],[139, 113],[138, 113],[136, 113],[135, 113],[134, 113],[133, 114],[131, 114],[130, 114],[128, 115],[127, 115],[126, 115],[125, 115],[124, 115],[122, 115],[121, 115],[120, 115],[118, 116],[115, 116],[113, 116],[111, 116],[109, 117],[106, 117],[103, 117],[102, 117],[100, 117],[98, 117],[97, 117],[95, 117],[94, 117],[93, 117],[92, 117],[91, 117],[90, 117],[89, 117],[88, 117],[87, 117],[86, 117],[85, 117],[84, 117],[83, 117],[84, 117],[85, 117],[87, 117],[88, 117],[89, 117],[90, 117],[92, 117],[93, 117],[95, 117],[97, 117],[99, 117],[101, 117],[103, 117],[104, 117],[105, 117],[106, 117],[107, 117],[108, 117],[109, 117],[110, 117],[112, 117],[113, 117],[114, 117],[116, 117],[117, 117],[118, 117],[119, 117],[120, 117],[121, 117],[123, 117],[124, 117],[125, 117],[126, 117],[127, 117],[129, 117],[130, 117],[131, 117],[133, 117],[134, 117],[135, 117],[136, 117],[137, 117],[138, 117],[139, 117],[140, 117],[141, 117],[142, 117],[143, 117],[145, 117],[146, 117],[147, 117],[148, 117],[149, 117],[150, 117],[149, 117],[148, 117],[147, 117],[146, 117],[144, 117],[143, 118],[142, 118],[141, 118],[140, 118],[139, 118],[138, 118],[136, 118],[135, 118],[132, 119],[131, 119],[130, 119],[129, 119],[127, 119],[126, 119],[124, 119],[123, 119],[122, 119],[121, 119],[119, 119],[118, 119],[117, 119],[115, 119],[114, 119],[113, 119],[112, 119],[111, 119],[110, 119],[109, 119],[108, 119],[107, 119],[106, 119],[107, 119],[108, 119],[109, 119],[110, 119],[112, 119],[113, 119],[114, 119],[115, 119],[116, 119],[117, 119],[118, 119],[119, 119],[120, 119],[121, 119],[122, 119],[123, 119],[124, 119],[125, 119],[126, 119],[127, 119],[127, 119],[127, 119],[127, 119]]"}),
|
||||
"camera": ("STRING", {"multiline": True, "default":"[[1,0,0,0,0,1,0,0,0,0,1,0.2]]"}),
|
||||
"traj": ("STRING", {"multiline": True, "default":"[[117, 102]]"}),
|
||||
"frame_length": ("INT", {"default": 16}),
|
||||
"steps": ("INT", {"default": 50}),
|
||||
"seed": ("INT", {"default": 1234}),
|
||||
@@ -406,7 +703,10 @@ class ImageSelector:
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"Motionctrl Sample":MotionctrlSample,
|
||||
"Motionctrl Sample Simple":MotionctrlSampleSimple,
|
||||
"Load Motion Camera Preset":LoadMotionCameraPreset,
|
||||
"Load Motion Traj Preset":LoadMotionTrajPreset,
|
||||
"Select Image Indices": ImageSelector
|
||||
"Select Image Indices": ImageSelector,
|
||||
"Load Motionctrl Checkpoint": MotionctrlLoader,
|
||||
"Motionctrl Cond": MotionctrlCond,
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
#This is an example that uses the websockets api to know when a prompt execution is done
|
||||
#Once the prompt execution is done it downloads the images using the /history endpoint
|
||||
import os
|
||||
import websocket #NOTE: websocket-client (https://github.com/websocket-client/websocket-client)
|
||||
import uuid
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
from flask import Flask, request, jsonify, render_template, session, abort
|
||||
from flask_socketio import SocketIO, join_room, leave_room,send, emit
|
||||
import secrets
|
||||
|
||||
from PIL import Image
|
||||
import io
|
||||
import base64
|
||||
|
||||
import time
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
|
||||
class FileCreatedHandler(FileSystemEventHandler):
|
||||
def on_created(self, event):
|
||||
# 处理文件创建完成的逻辑
|
||||
file_path = event.src_path
|
||||
file_name = os.path.basename(file_path)
|
||||
print(f"New file created: {file_name}")
|
||||
|
||||
time.sleep(.1)
|
||||
with open(file_path, 'rb') as fr:
|
||||
image_data=fr.read()
|
||||
b64img=base64.b64encode(image_data).decode('utf-8')
|
||||
socketio.emit('server_response',{'b64img':b64img})
|
||||
|
||||
folder_to_watch = "/home/admin/ComfyUI/output/motionctrl" # 要监控的文件夹路径
|
||||
|
||||
event_handler = FileCreatedHandler() # 创建我们刚才定义的自定义处理类的实例
|
||||
observer = Observer()
|
||||
observer.schedule(event_handler, folder_to_watch, recursive=False)
|
||||
observer.start()
|
||||
|
||||
server_address = "127.0.0.1:8188"
|
||||
client_id = str(uuid.uuid4())
|
||||
|
||||
def queue_prompt(prompt):
|
||||
p = {"prompt": prompt, "client_id": client_id}
|
||||
data = json.dumps(p).encode('utf-8')
|
||||
req = urllib.request.Request("http://{}/prompt".format(server_address), data=data)
|
||||
return json.loads(urllib.request.urlopen(req).read())
|
||||
|
||||
def get_image(filename, subfolder, folder_type):
|
||||
data = {"filename": filename, "subfolder": subfolder, "type": folder_type}
|
||||
url_values = urllib.parse.urlencode(data)
|
||||
with urllib.request.urlopen("http://{}/view?{}".format(server_address, url_values)) as response:
|
||||
return response.read()
|
||||
|
||||
def get_history(prompt_id):
|
||||
with urllib.request.urlopen("http://{}/history/{}".format(server_address, prompt_id)) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
def get_images(ws, prompt):
|
||||
prompt_id = queue_prompt(prompt)['prompt_id']
|
||||
output_images = {}
|
||||
'''
|
||||
while True:
|
||||
out = ws.recv()
|
||||
if isinstance(out, str):
|
||||
message = json.loads(out)
|
||||
if message['type'] == 'executing':
|
||||
data = message['data']
|
||||
if data['node'] is None and data['prompt_id'] == prompt_id:
|
||||
break #Execution is done
|
||||
else:
|
||||
continue #previews are binary data
|
||||
|
||||
history = get_history(prompt_id)[prompt_id]
|
||||
for o in history['outputs']:
|
||||
for node_id in history['outputs']:
|
||||
node_output = history['outputs'][node_id]
|
||||
if 'images' in node_output:
|
||||
images_output = []
|
||||
for image in node_output['images']:
|
||||
image_data = get_image(image['filename'], image['subfolder'], image['type'])
|
||||
images_output.append(image_data)
|
||||
output_images[node_id] = images_output
|
||||
|
||||
return output_images
|
||||
'''
|
||||
|
||||
prompt={}
|
||||
with open('./workflow_api_motionctrl_turbo.json') as fr:
|
||||
prompt = json.load(fr)
|
||||
|
||||
ws = websocket.WebSocket()
|
||||
ws.connect("ws://{}/ws?clientId={}".format(server_address, client_id))
|
||||
|
||||
#Commented out code to display the output images:
|
||||
|
||||
# for node_id in images:
|
||||
# for image_data in images[node_id]:
|
||||
# from PIL import Image
|
||||
# import io
|
||||
# image = Image.open(io.BytesIO(image_data))
|
||||
# image.show()
|
||||
|
||||
app = Flask(__name__, template_folder=os.path.abspath('.'), static_folder='assets')
|
||||
app.secret_key = secrets.token_hex(16)
|
||||
|
||||
socketio = SocketIO(app, cors_allowed_origins='*')
|
||||
connected_sids = set() # 存放已连接的客户端
|
||||
|
||||
#后端程序
|
||||
lockroom='None'
|
||||
@socketio.on('connect')
|
||||
def on_connect():
|
||||
connected_sids.add(request.sid)
|
||||
print(f'{request.sid} 已连接')
|
||||
socketio.start_background_task(background_thread_heartbeat)
|
||||
|
||||
@socketio.on('disconnect')
|
||||
def on_disconnect():
|
||||
connected_sids.remove(request.sid)
|
||||
print(f'{request.sid} 已断开')
|
||||
|
||||
@socketio.on('message')
|
||||
def handle_message(message):
|
||||
"""收消息"""
|
||||
print(f'message:{request.sid} {message}')
|
||||
json.loads(message)
|
||||
|
||||
@socketio.on('camera_poses')
|
||||
def handle_message(camera_poses):
|
||||
print(f'camera_poses:{request.sid} {camera_poses}')
|
||||
cams=json.loads(camera_poses["camera_poses"])
|
||||
trajs=json.loads(camera_poses["trajs"])
|
||||
if len(cams)>1 and len(trajs)>1:
|
||||
prompt["60"]["inputs"]["infer_mode"] = "control both camera and object motion"
|
||||
elif len(trajs)>1:
|
||||
prompt["60"]["inputs"]["infer_mode"] = "control object trajectory"
|
||||
else:
|
||||
prompt["60"]["inputs"]["infer_mode"] = "control camera poses"
|
||||
|
||||
prompt["60"]["inputs"]["prompt"] = camera_poses["prompt"]
|
||||
prompt["60"]["inputs"]["camera"] = camera_poses["camera_poses"]
|
||||
prompt["60"]["inputs"]["traj"] = camera_poses["trajs"]
|
||||
images = get_images(ws, prompt)
|
||||
'''
|
||||
for node_id in images:
|
||||
for image_data in images[node_id]:
|
||||
b64img=base64.b64encode(image_data).decode('utf-8')
|
||||
socketio.emit('server_response',{'b64img':b64img}, to=camera_poses["roomid"])
|
||||
'''
|
||||
|
||||
@socketio.on('server_reconnect')
|
||||
def server_reconnect(message):
|
||||
print(f'server_reconnect:{request.sid} {message}')
|
||||
join_room(message['roomid'])
|
||||
|
||||
def background_thread_heartbeat():
|
||||
global lockroom
|
||||
while True:
|
||||
socketio.emit('server_response',{'lockroom':lockroom})
|
||||
socketio.sleep(5)
|
||||
|
||||
|
||||
@app.route('/')
|
||||
def index():
|
||||
session['user'] = None
|
||||
return render_template('index.html')
|
||||
|
||||
if __name__ == '__main__':
|
||||
socketio.run(app, host='0.0.0.0', port=5017, debug=True, allow_unsafe_werkzeug=True)
|
||||
#app.run(host='0.0.0.0', port=5017)
|
||||
@@ -0,0 +1,26 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
|
||||
<link type="text/css" rel="stylesheet" href="main.css">
|
||||
<title>CAMERA MOTION DESIGNER</title>
|
||||
<style>
|
||||
body { margin: 0; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="info">
|
||||
<button id="btn_translate">Translate</button>
|
||||
<button id="btn_rotate">Rotate</button>
|
||||
<input id="txt_prompt" value="a rose swaying in the wind" />
|
||||
<button id="btn_addpoint" style="display:none;">Add Point</button>
|
||||
<!--"W" translate | "E" rotate--><br />
|
||||
<!--textarea id="tb_result" style="width:256px; height:128px;">[]</textarea><br/-->
|
||||
<button id="btn_startrt">Start Real Time MotionCtrl</button><br/>
|
||||
<div style="width:256px;height:256px;display:none;" class="imageContainer"></div>
|
||||
<canvas id="canvas" height="256" width="256" style="border:1px dotted gray;"></canvas>
|
||||
</div>
|
||||
<script type="module" src="main.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,91 @@
|
||||
body {
|
||||
margin: 0;
|
||||
background-color: #000;
|
||||
color: #fff;
|
||||
font-family: Monospace;
|
||||
font-size: 13px;
|
||||
line-height: 24px;
|
||||
overscroll-behavior: none;
|
||||
}
|
||||
|
||||
a {
|
||||
color: #ff0;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
button {
|
||||
cursor: pointer;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
#info {
|
||||
position: absolute;
|
||||
top: 0px;
|
||||
left: 0px;
|
||||
padding: 10px;
|
||||
box-sizing: border-box;
|
||||
text-align: center;
|
||||
/*-moz-user-select: none;
|
||||
-webkit-user-select: none;
|
||||
-ms-user-select: none;
|
||||
user-select: none;
|
||||
pointer-events: none;*/
|
||||
z-index: 1; /* TODO Solve this in HTML */
|
||||
}
|
||||
|
||||
a, button, input, select {
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
.lil-gui {
|
||||
z-index: 2 !important; /* TODO Solve this in HTML */
|
||||
}
|
||||
|
||||
@media all and ( max-width: 640px ) {
|
||||
.lil-gui.root {
|
||||
right: auto;
|
||||
top: auto;
|
||||
max-height: 50%;
|
||||
max-width: 80%;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
}
|
||||
}
|
||||
|
||||
#overlay {
|
||||
position: absolute;
|
||||
font-size: 16px;
|
||||
z-index: 2;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex-direction: column;
|
||||
background: rgba(0,0,0,0.7);
|
||||
}
|
||||
|
||||
#overlay button {
|
||||
background: transparent;
|
||||
border: 0;
|
||||
border: 1px solid rgb(255, 255, 255);
|
||||
border-radius: 4px;
|
||||
color: #ffffff;
|
||||
padding: 12px 18px;
|
||||
text-transform: uppercase;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
#notSupported {
|
||||
width: 50%;
|
||||
margin: auto;
|
||||
background-color: #f00;
|
||||
margin-top: 20px;
|
||||
padding: 10px;
|
||||
}
|
||||
+392
@@ -0,0 +1,392 @@
|
||||
import * as THREE from 'three';
|
||||
|
||||
import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
|
||||
import { TransformControls } from 'three/examples/jsm/controls/TransformControls.js';
|
||||
import { io } from "https://cdn.socket.io/4.7.2/socket.io.esm.min.js";
|
||||
|
||||
let cameraPersp, cameraPerspTransform, currentCamera, cameraPerspTransformHelper;
|
||||
let scene, renderer, control, orbit;
|
||||
|
||||
let roomid=new Date().getTime();
|
||||
|
||||
let cposes=[];
|
||||
let cmatrix=[];
|
||||
let userDrawnPixels1024=[];
|
||||
|
||||
var canvas,ctx;
|
||||
var mouseX,mouseY,moving, mouseDown=0;
|
||||
function getTouchPos(e) {
|
||||
if (!e)
|
||||
var e = event;
|
||||
|
||||
if(e.touches) {
|
||||
if (e.touches.length == 1) {
|
||||
var touch = e.touches[0];
|
||||
mouseX=touch.pageX-touch.target.offsetLeft;
|
||||
mouseY=touch.pageY-touch.target.offsetTop;
|
||||
}
|
||||
}
|
||||
}
|
||||
function sketchpad_mouseDown(e) {
|
||||
getMousePos(e);
|
||||
userDrawnPixelsPush([mouseX, mouseY]);
|
||||
mouseDown=1;
|
||||
}
|
||||
function mouseOrTouchUp(e) {
|
||||
mouseDown=0;
|
||||
moving=0;
|
||||
|
||||
if(!mouseX){
|
||||
getMousePos(e);
|
||||
userDrawnPixelsPush([mouseX, mouseY]);
|
||||
moving=1;
|
||||
}else{
|
||||
getMousePos(e);
|
||||
userDrawnPixelsPush([mouseX, mouseY]);
|
||||
moving=1;
|
||||
}
|
||||
}
|
||||
|
||||
function sketchpad_mouseMove(e) {
|
||||
getMousePos(e);
|
||||
|
||||
if (mouseDown==1) {
|
||||
userDrawnPixelsPush([mouseX, mouseY]);
|
||||
moving=1;
|
||||
}
|
||||
}
|
||||
|
||||
function getMousePos(e) {
|
||||
if (!e)
|
||||
var e = event;
|
||||
|
||||
if (e.offsetX) {
|
||||
mouseX = e.offsetX;
|
||||
mouseY = e.offsetY;
|
||||
}
|
||||
else if (e.layerX) {
|
||||
mouseX = e.layerX;
|
||||
mouseY = e.layerY;
|
||||
}
|
||||
}
|
||||
|
||||
function sketchpad_touchStart(e) {
|
||||
getTouchPos(e);
|
||||
userDrawnPixelsPush([mouseX, mouseY]);
|
||||
event.preventDefault();
|
||||
|
||||
moving=1;
|
||||
mouseDown=1;
|
||||
}
|
||||
|
||||
function sketchpad_touchMove(e) {
|
||||
getTouchPos(e);
|
||||
userDrawnPixelsPush([mouseX, mouseY]);
|
||||
|
||||
event.preventDefault();
|
||||
}
|
||||
|
||||
function userDrawnPixelsPush(point){
|
||||
if(userDrawnPixels1024.length==0&&point[0]!=null){
|
||||
userDrawnPixels1024.push([point[0]*4,point[1]*4]);
|
||||
}else{
|
||||
if(point[0]!=null&&(userDrawnPixels1024[userDrawnPixels1024.length-1][0]!=point[0]*4||userDrawnPixels1024[userDrawnPixels1024.length-1][1]!=point[1]*4)){
|
||||
userDrawnPixels1024.push([point[0]*4,point[1]*4]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var socket = io();
|
||||
socket.on('connect', function() {
|
||||
socket.emit('server_reconnect', {roomid: roomid});
|
||||
});
|
||||
|
||||
|
||||
socket.on("server_response", function (msg,ack) {
|
||||
console.log(msg);
|
||||
//接收到后端发送过来的消息
|
||||
var b64img = msg.b64img;
|
||||
if(!b64img)return;
|
||||
|
||||
var image = new Image();
|
||||
image.src = 'data:image/jpeg;base64,' + b64img;
|
||||
|
||||
image.onload=function(){
|
||||
ctx.drawImage(image,0,0,256,256);
|
||||
drawDot(ctx,userDrawnPixels1024[userDrawnPixels1024.length-1][0]/4,userDrawnPixels1024[userDrawnPixels1024.length-1][1]/4,6);
|
||||
};
|
||||
});
|
||||
|
||||
init();
|
||||
render();
|
||||
|
||||
|
||||
|
||||
function detect_change(){
|
||||
var matrix=cameraPerspTransform.matrix.elements;
|
||||
matrix[3]=cameraPerspTransform.position.x;
|
||||
matrix[7]=cameraPerspTransform.position.y;
|
||||
matrix[11]=cameraPerspTransform.position.z;
|
||||
matrix=matrix.slice(0, 12);
|
||||
|
||||
if(!userDrawnPixels1024.length){
|
||||
userDrawnPixels1024=[[128*4,128*4]];
|
||||
}
|
||||
|
||||
if(JSON.stringify(cmatrix)!=JSON.stringify(matrix)){
|
||||
if(userDrawnPixels1024.length>=16){
|
||||
socket.emit('camera_poses', {roomid:roomid,prompt:document.getElementById('txt_prompt').value,camera_poses:JSON.stringify(cposes),trajs:JSON.stringify(userDrawnPixels1024)});
|
||||
userDrawnPixels1024=[userDrawnPixels1024[userDrawnPixels1024.length-1]];
|
||||
cposes=[matrix];
|
||||
}else{
|
||||
cposes.push(matrix);
|
||||
}
|
||||
cmatrix=matrix;
|
||||
}else{
|
||||
if(cposes.length>1||userDrawnPixels1024.length>=16){
|
||||
socket.emit('camera_poses', {roomid:roomid,prompt:document.getElementById('txt_prompt').value,camera_poses:JSON.stringify(cposes),trajs:JSON.stringify(userDrawnPixels1024)});
|
||||
userDrawnPixels1024=[userDrawnPixels1024[userDrawnPixels1024.length-1]];
|
||||
}
|
||||
|
||||
cposes=[matrix];
|
||||
}
|
||||
|
||||
setTimeout(function(){
|
||||
detect_change();
|
||||
},100);
|
||||
}
|
||||
|
||||
function drawDot(ctx, x, y, size) {
|
||||
ctx.fillStyle = "lightgrey";
|
||||
|
||||
ctx.beginPath();
|
||||
// Draw a filled circle
|
||||
ctx.arc(x, y, size, 0, Math.PI*2, true);
|
||||
ctx.closePath();
|
||||
ctx.fill();
|
||||
}
|
||||
|
||||
function init() {
|
||||
canvas = document.getElementById('canvas');
|
||||
|
||||
canvas.width = 256;
|
||||
canvas.height = 256;
|
||||
|
||||
if (canvas.getContext)
|
||||
ctx = canvas.getContext('2d');
|
||||
drawDot(ctx,128,128,6);
|
||||
|
||||
if (ctx) {
|
||||
// React to mouse events on the canvas, and mouseup on the entire document
|
||||
canvas.addEventListener('mousedown', sketchpad_mouseDown, false);
|
||||
canvas.addEventListener('mousemove', sketchpad_mouseMove, false);
|
||||
canvas.addEventListener('mouseup', mouseOrTouchUp, false);
|
||||
|
||||
// React to touch events on the canvas
|
||||
canvas.addEventListener('touchstart', sketchpad_touchStart, false);
|
||||
canvas.addEventListener('touchmove', sketchpad_touchMove, false);
|
||||
canvas.addEventListener('touchend', mouseOrTouchUp, false);
|
||||
}
|
||||
|
||||
renderer = new THREE.WebGLRenderer( { antialias: true } );
|
||||
renderer.setPixelRatio( window.devicePixelRatio );
|
||||
renderer.setSize( window.innerWidth, window.innerHeight );
|
||||
document.body.appendChild( renderer.domElement );
|
||||
|
||||
const aspect = window.innerWidth / window.innerHeight;
|
||||
|
||||
cameraPersp = new THREE.PerspectiveCamera( 50, aspect, 0.01, 30000 );
|
||||
cameraPerspTransform = new THREE.PerspectiveCamera( 50, aspect, 0.01, 30000 );
|
||||
|
||||
cameraPerspTransformHelper = new THREE.CameraHelper( cameraPerspTransform );
|
||||
|
||||
currentCamera = cameraPersp;
|
||||
|
||||
currentCamera.position.set( 5, 2.5, 5 );
|
||||
|
||||
scene = new THREE.Scene();
|
||||
scene.add( new THREE.GridHelper( 5, 10, 0x888888, 0x444444 ) );
|
||||
|
||||
const ambientLight = new THREE.AmbientLight( 0xffffff );
|
||||
scene.add( ambientLight );
|
||||
|
||||
const light = new THREE.DirectionalLight( 0xffffff, 4 );
|
||||
light.position.set( 1, 1, 1 );
|
||||
scene.add( light );
|
||||
|
||||
const texture = new THREE.TextureLoader().load( 'textures/crate.gif', render );
|
||||
texture.colorSpace = THREE.SRGBColorSpace;
|
||||
texture.anisotropy = renderer.capabilities.getMaxAnisotropy();
|
||||
|
||||
const geometry = new THREE.BoxGeometry();
|
||||
const material = new THREE.MeshLambertMaterial( { map: texture } );
|
||||
|
||||
orbit = new OrbitControls( currentCamera, renderer.domElement );
|
||||
orbit.update();
|
||||
orbit.addEventListener( 'change', render );
|
||||
|
||||
control = new TransformControls( currentCamera, renderer.domElement );
|
||||
control.addEventListener( 'change', render );
|
||||
|
||||
control.addEventListener( 'dragging-changed', function ( event ) {
|
||||
|
||||
orbit.enabled = ! event.value;
|
||||
|
||||
} );
|
||||
|
||||
const mesh = new THREE.Mesh( geometry, material );
|
||||
scene.add( cameraPerspTransform );
|
||||
|
||||
control.attach( cameraPerspTransform );
|
||||
scene.add( control );
|
||||
scene.add( cameraPerspTransformHelper );
|
||||
|
||||
document.getElementById('btn_translate').addEventListener('click',function(){
|
||||
control.setMode( 'translate' );
|
||||
});
|
||||
|
||||
document.getElementById('btn_rotate').addEventListener('click',function(){
|
||||
control.setMode( 'rotate' );
|
||||
});
|
||||
|
||||
document.getElementById('btn_addpoint').addEventListener('click',function(){
|
||||
var ret=JSON.parse(document.getElementById('tb_result').value);
|
||||
var matrix=cameraPerspTransform.matrix.elements;
|
||||
matrix[3]=cameraPerspTransform.position.x;
|
||||
matrix[7]=cameraPerspTransform.position.y;
|
||||
matrix[11]=cameraPerspTransform.position.z;
|
||||
matrix=matrix.slice(0, 12);
|
||||
ret.push(matrix);
|
||||
document.getElementById('tb_result').value=JSON.stringify(ret);
|
||||
});
|
||||
|
||||
document.getElementById('btn_startrt').addEventListener('click',function(){
|
||||
detect_change();
|
||||
});
|
||||
|
||||
window.addEventListener( 'resize', onWindowResize );
|
||||
|
||||
window.addEventListener( 'keydown', function ( event ) {
|
||||
|
||||
switch ( event.keyCode ) {
|
||||
|
||||
case 81: // Q
|
||||
control.setSpace( control.space === 'local' ? 'world' : 'local' );
|
||||
break;
|
||||
|
||||
case 16: // Shift
|
||||
control.setTranslationSnap( 100 );
|
||||
control.setRotationSnap( THREE.MathUtils.degToRad( 15 ) );
|
||||
control.setScaleSnap( 0.25 );
|
||||
break;
|
||||
|
||||
case 87: // W
|
||||
control.setMode( 'translate' );
|
||||
break;
|
||||
|
||||
case 69: // E
|
||||
control.setMode( 'rotate' );
|
||||
break;
|
||||
|
||||
case 82: // R
|
||||
control.setMode( 'scale' );
|
||||
break;
|
||||
|
||||
case 67: // C
|
||||
const position = currentCamera.position.clone();
|
||||
|
||||
currentCamera = currentCamera.isPerspectiveCamera ? cameraPerspTransform : cameraPersp;
|
||||
currentCamera.position.copy( position );
|
||||
|
||||
orbit.object = currentCamera;
|
||||
control.camera = currentCamera;
|
||||
|
||||
currentCamera.lookAt( orbit.target.x, orbit.target.y, orbit.target.z );
|
||||
onWindowResize();
|
||||
break;
|
||||
|
||||
case 86: // V
|
||||
const randomFoV = Math.random() + 0.1;
|
||||
const randomZoom = Math.random() + 0.1;
|
||||
|
||||
cameraPersp.fov = randomFoV * 160;
|
||||
cameraPerspTransform.bottom = - randomFoV * 500;
|
||||
cameraPerspTransform.top = randomFoV * 500;
|
||||
|
||||
cameraPersp.zoom = randomZoom * 5;
|
||||
cameraPerspTransform.zoom = randomZoom * 5;
|
||||
onWindowResize();
|
||||
break;
|
||||
|
||||
case 187:
|
||||
case 107: // +, =, num+
|
||||
control.setSize( control.size + 0.1 );
|
||||
break;
|
||||
|
||||
case 189:
|
||||
case 109: // -, _, num-
|
||||
control.setSize( Math.max( control.size - 0.1, 0.1 ) );
|
||||
break;
|
||||
|
||||
case 88: // X
|
||||
control.showX = ! control.showX;
|
||||
break;
|
||||
|
||||
case 89: // Y
|
||||
control.showY = ! control.showY;
|
||||
break;
|
||||
|
||||
case 90: // Z
|
||||
control.showZ = ! control.showZ;
|
||||
break;
|
||||
|
||||
case 32: // Spacebar
|
||||
control.enabled = ! control.enabled;
|
||||
break;
|
||||
|
||||
case 27: // Esc
|
||||
control.reset();
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
} );
|
||||
|
||||
window.addEventListener( 'keyup', function ( event ) {
|
||||
|
||||
switch ( event.keyCode ) {
|
||||
|
||||
case 16: // Shift
|
||||
control.setTranslationSnap( null );
|
||||
control.setRotationSnap( null );
|
||||
control.setScaleSnap( null );
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
} );
|
||||
|
||||
}
|
||||
|
||||
function onWindowResize() {
|
||||
|
||||
const aspect = window.innerWidth / window.innerHeight;
|
||||
|
||||
cameraPersp.aspect = aspect;
|
||||
cameraPersp.updateProjectionMatrix();
|
||||
|
||||
cameraPerspTransform.left = cameraPerspTransform.bottom * aspect;
|
||||
cameraPerspTransform.right = cameraPerspTransform.top * aspect;
|
||||
cameraPerspTransform.updateProjectionMatrix();
|
||||
|
||||
renderer.setSize( window.innerWidth, window.innerHeight );
|
||||
|
||||
render();
|
||||
|
||||
}
|
||||
|
||||
function render() {
|
||||
|
||||
renderer.render( scene, currentCamera );
|
||||
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"three": "^0.160.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"vite": "^5.0.11"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"56": {
|
||||
"inputs": {
|
||||
"ckpt_name": "motionctrl.pth",
|
||||
"frame_length": 16
|
||||
},
|
||||
"class_type": "Load Motionctrl Checkpoint"
|
||||
},
|
||||
"60": {
|
||||
"inputs": {
|
||||
"prompt": "a rose swaying in the wind",
|
||||
"camera": "[[1,0,0,0,0,1,0,0,0,0,1,0.2]]",
|
||||
"traj": "[[117, 102]]",
|
||||
"infer_mode": "control camera poses",
|
||||
"context_overlap": 4,
|
||||
"model": [
|
||||
"56",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "Motionctrl Cond"
|
||||
},
|
||||
"61": {
|
||||
"inputs": {
|
||||
"steps": 20,
|
||||
"seed": 1647,
|
||||
"context_overlap": [
|
||||
"60",
|
||||
7
|
||||
],
|
||||
"traj_tool": "https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html",
|
||||
"draw_traj_dot": false,
|
||||
"draw_camera_dot": false,
|
||||
"model": [
|
||||
"56",
|
||||
0
|
||||
],
|
||||
"clip": [
|
||||
"56",
|
||||
1
|
||||
],
|
||||
"vae": [
|
||||
"56",
|
||||
2
|
||||
],
|
||||
"ddim_sampler": [
|
||||
"56",
|
||||
3
|
||||
],
|
||||
"positive": [
|
||||
"60",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"60",
|
||||
1
|
||||
],
|
||||
"traj_list": [
|
||||
"60",
|
||||
2
|
||||
],
|
||||
"rt_list": [
|
||||
"60",
|
||||
3
|
||||
],
|
||||
"traj": [
|
||||
"60",
|
||||
4
|
||||
],
|
||||
"rt": [
|
||||
"60",
|
||||
5
|
||||
],
|
||||
"noise_shape": [
|
||||
"60",
|
||||
6
|
||||
]
|
||||
},
|
||||
"class_type": "Motionctrl Sample Simple"
|
||||
},
|
||||
"62": {
|
||||
"inputs": {
|
||||
"filename_prefix": "motionctrl/motionctrl",
|
||||
"images": [
|
||||
"61",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "SaveImage"
|
||||
}
|
||||
}
|
||||
+438
-132
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"last_node_id": 26,
|
||||
"last_link_id": 34,
|
||||
"last_node_id": 30,
|
||||
"last_link_id": 56,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 9,
|
||||
@@ -14,7 +14,7 @@
|
||||
"1": 82
|
||||
},
|
||||
"flags": {},
|
||||
"order": 13,
|
||||
"order": 15,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -52,7 +52,7 @@
|
||||
"1": 190
|
||||
},
|
||||
"flags": {},
|
||||
"order": 9,
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -109,7 +109,7 @@
|
||||
"1": 166
|
||||
},
|
||||
"flags": {},
|
||||
"order": 15,
|
||||
"order": 17,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -258,7 +258,7 @@
|
||||
"1": 200
|
||||
},
|
||||
"flags": {},
|
||||
"order": 11,
|
||||
"order": 10,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -344,7 +344,7 @@
|
||||
"1": 46
|
||||
},
|
||||
"flags": {},
|
||||
"order": 18,
|
||||
"order": 20,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -384,7 +384,7 @@
|
||||
539
|
||||
],
|
||||
"flags": {},
|
||||
"order": 19,
|
||||
"order": 21,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -411,7 +411,7 @@
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "AnimateDiff_00451.mp4",
|
||||
"filename": "AnimateDiff_00468.mp4",
|
||||
"subfolder": "",
|
||||
"type": "output",
|
||||
"format": "video/h264-mp4"
|
||||
@@ -487,7 +487,7 @@
|
||||
"1": 246
|
||||
},
|
||||
"flags": {},
|
||||
"order": 16,
|
||||
"order": 18,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -512,7 +512,7 @@
|
||||
"1": 262
|
||||
},
|
||||
"flags": {},
|
||||
"order": 17,
|
||||
"order": 19,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -550,7 +550,7 @@
|
||||
"Node name for S&R": "KSampler"
|
||||
},
|
||||
"widgets_values": [
|
||||
557139385238250,
|
||||
540393992754865,
|
||||
"randomize",
|
||||
8,
|
||||
2,
|
||||
@@ -571,13 +571,13 @@
|
||||
539
|
||||
],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"order": 13,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 4
|
||||
"link": 48
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
@@ -598,7 +598,7 @@
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "AnimateDiff_00450.gif",
|
||||
"filename": "AnimateDiff_00467.gif",
|
||||
"subfolder": "",
|
||||
"type": "output",
|
||||
"format": "image/gif"
|
||||
@@ -638,45 +638,6 @@
|
||||
"0,4,7,11,15,19,23,27,31"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 19,
|
||||
"type": "CLIPTextEncode",
|
||||
"pos": [
|
||||
754,
|
||||
1150
|
||||
],
|
||||
"size": {
|
||||
"0": 400,
|
||||
"1": 200
|
||||
},
|
||||
"flags": {},
|
||||
"order": 10,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": 15
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CONDITIONING",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
24
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CLIPTextEncode"
|
||||
},
|
||||
"widgets_values": [
|
||||
"a leaf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"type": "EmptyLatentImage",
|
||||
@@ -711,58 +672,19 @@
|
||||
32
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 26,
|
||||
"type": "Select Image Indices",
|
||||
"pos": [
|
||||
475,
|
||||
114
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 58
|
||||
},
|
||||
"flags": {},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 33
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
34
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Select Image Indices"
|
||||
},
|
||||
"widgets_values": [
|
||||
"0,4,7,11,15,19,23,27,31"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "ImageScale",
|
||||
"pos": [
|
||||
474,
|
||||
247
|
||||
209,
|
||||
199
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 130
|
||||
},
|
||||
"flags": {},
|
||||
"order": 12,
|
||||
"order": 14,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -797,15 +719,15 @@
|
||||
"id": 25,
|
||||
"type": "FakeScribblePreprocessor",
|
||||
"pos": [
|
||||
469,
|
||||
457
|
||||
232,
|
||||
449
|
||||
],
|
||||
"size": {
|
||||
"0": 319.20001220703125,
|
||||
"1": 82
|
||||
},
|
||||
"flags": {},
|
||||
"order": 14,
|
||||
"order": 16,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
@@ -835,57 +757,337 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "Motionctrl Sample",
|
||||
"id": 26,
|
||||
"type": "Select Image Indices",
|
||||
"pos": [
|
||||
11,
|
||||
110
|
||||
207,
|
||||
35
|
||||
],
|
||||
"size": {
|
||||
"0": 400,
|
||||
"1": 344
|
||||
"0": 315,
|
||||
"1": 58
|
||||
},
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"order": 12,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 47
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
4,
|
||||
33
|
||||
34
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Motionctrl Sample"
|
||||
"Node name for S&R": "Select Image Indices"
|
||||
},
|
||||
"widgets_values": [
|
||||
"0,4,7,11,15,19,23,27,31"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 27,
|
||||
"type": "Load Motionctrl Checkpoint",
|
||||
"pos": [
|
||||
-660,
|
||||
6
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 142
|
||||
},
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MOTIONCTRL",
|
||||
"links": [
|
||||
36,
|
||||
50
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
},
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "EMBEDDER",
|
||||
"links": [
|
||||
37
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 1
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"links": [
|
||||
38
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 2
|
||||
},
|
||||
{
|
||||
"name": "ddim_sampler",
|
||||
"type": "SAMPLER",
|
||||
"links": [
|
||||
39
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 3
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Load Motionctrl Checkpoint"
|
||||
},
|
||||
"widgets_values": [
|
||||
"motionctrl.pth",
|
||||
32
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 29,
|
||||
"type": "Motionctrl Sample Simple",
|
||||
"pos": [
|
||||
-181,
|
||||
-7
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 378
|
||||
},
|
||||
"flags": {},
|
||||
"order": 11,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MOTIONCTRL",
|
||||
"link": 36
|
||||
},
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "EMBEDDER",
|
||||
"link": 37
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": 38
|
||||
},
|
||||
{
|
||||
"name": "ddim_sampler",
|
||||
"type": "SAMPLER",
|
||||
"link": 39
|
||||
},
|
||||
{
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"link": 49
|
||||
},
|
||||
{
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"link": 51
|
||||
},
|
||||
{
|
||||
"name": "traj_list",
|
||||
"type": "TRAJ_LIST",
|
||||
"link": 52
|
||||
},
|
||||
{
|
||||
"name": "rt_list",
|
||||
"type": "RT_LIST",
|
||||
"link": 53
|
||||
},
|
||||
{
|
||||
"name": "traj",
|
||||
"type": "TRAJ_FEATURES",
|
||||
"link": 54
|
||||
},
|
||||
{
|
||||
"name": "rt",
|
||||
"type": "RT",
|
||||
"link": 55
|
||||
},
|
||||
{
|
||||
"name": "noise_shape",
|
||||
"type": "NOISE_SHAPE",
|
||||
"link": 56
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
47,
|
||||
48
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Motionctrl Sample Simple"
|
||||
},
|
||||
"widgets_values": [
|
||||
30,
|
||||
1874,
|
||||
"randomize",
|
||||
"https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html",
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "Motionctrl Cond",
|
||||
"pos": [
|
||||
-702,
|
||||
332
|
||||
],
|
||||
"size": {
|
||||
"0": 400,
|
||||
"1": 320
|
||||
},
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MOTIONCTRL",
|
||||
"link": 50
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
49
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
},
|
||||
{
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
51
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 1
|
||||
},
|
||||
{
|
||||
"name": "traj_list",
|
||||
"type": "TRAJ_LIST",
|
||||
"links": [
|
||||
52
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 2
|
||||
},
|
||||
{
|
||||
"name": "rt_list",
|
||||
"type": "RT_LIST",
|
||||
"links": [
|
||||
53
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 3
|
||||
},
|
||||
{
|
||||
"name": "traj",
|
||||
"type": "TRAJ_FEATURES",
|
||||
"links": [
|
||||
54
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 4
|
||||
},
|
||||
{
|
||||
"name": "rt",
|
||||
"type": "RT",
|
||||
"links": [
|
||||
55
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 5
|
||||
},
|
||||
{
|
||||
"name": "noise_shape",
|
||||
"type": "NOISE_SHAPE",
|
||||
"links": [
|
||||
56
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 6
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Motionctrl Cond"
|
||||
},
|
||||
"widgets_values": [
|
||||
"a girl jump",
|
||||
"[[1,0,0,-0.2,0,1,0,0,0,0,1,0]]",
|
||||
"[[84,752],[96,556],[128,424],[156,320],[184,252],[232,188],[272,144],[328,108],[384,92],[440,92],[488,88],[620,116],[680,116],[764,132],[804,144],[872,252],[884,344],[916,440],[904,568],[888,628],[828,708],[760,764],[704,816],[616,848],[516,876],[420,876],[336,876],[288,868],[196,844],[136,816],[88,792],[68,740]]",
|
||||
32,
|
||||
50,
|
||||
2048,
|
||||
"fixed",
|
||||
"https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html",
|
||||
false,
|
||||
false
|
||||
"control both camera and object motion"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 19,
|
||||
"type": "CLIPTextEncode",
|
||||
"pos": [
|
||||
754,
|
||||
1150
|
||||
],
|
||||
"size": {
|
||||
"0": 400,
|
||||
"1": 200
|
||||
},
|
||||
"flags": {},
|
||||
"order": 9,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "CLIP",
|
||||
"link": 15
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "CONDITIONING",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
24
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "CLIPTextEncode"
|
||||
},
|
||||
"widgets_values": [
|
||||
"a girl jump"
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
4,
|
||||
3,
|
||||
0,
|
||||
6,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
6,
|
||||
8,
|
||||
@@ -1054,14 +1256,6 @@
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
33,
|
||||
3,
|
||||
0,
|
||||
26,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
34,
|
||||
26,
|
||||
@@ -1069,6 +1263,118 @@
|
||||
8,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
36,
|
||||
27,
|
||||
0,
|
||||
29,
|
||||
0,
|
||||
"MOTIONCTRL"
|
||||
],
|
||||
[
|
||||
37,
|
||||
27,
|
||||
1,
|
||||
29,
|
||||
1,
|
||||
"EMBEDDER"
|
||||
],
|
||||
[
|
||||
38,
|
||||
27,
|
||||
2,
|
||||
29,
|
||||
2,
|
||||
"VAE"
|
||||
],
|
||||
[
|
||||
39,
|
||||
27,
|
||||
3,
|
||||
29,
|
||||
3,
|
||||
"SAMPLER"
|
||||
],
|
||||
[
|
||||
47,
|
||||
29,
|
||||
0,
|
||||
26,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
48,
|
||||
29,
|
||||
0,
|
||||
6,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
49,
|
||||
30,
|
||||
0,
|
||||
29,
|
||||
4,
|
||||
"CONDITIONING"
|
||||
],
|
||||
[
|
||||
50,
|
||||
27,
|
||||
0,
|
||||
30,
|
||||
0,
|
||||
"MOTIONCTRL"
|
||||
],
|
||||
[
|
||||
51,
|
||||
30,
|
||||
1,
|
||||
29,
|
||||
5,
|
||||
"CONDITIONING"
|
||||
],
|
||||
[
|
||||
52,
|
||||
30,
|
||||
2,
|
||||
29,
|
||||
6,
|
||||
"TRAJ_LIST"
|
||||
],
|
||||
[
|
||||
53,
|
||||
30,
|
||||
3,
|
||||
29,
|
||||
7,
|
||||
"RT_LIST"
|
||||
],
|
||||
[
|
||||
54,
|
||||
30,
|
||||
4,
|
||||
29,
|
||||
8,
|
||||
"TRAJ_FEATURES"
|
||||
],
|
||||
[
|
||||
55,
|
||||
30,
|
||||
5,
|
||||
29,
|
||||
9,
|
||||
"RT"
|
||||
],
|
||||
[
|
||||
56,
|
||||
30,
|
||||
6,
|
||||
29,
|
||||
10,
|
||||
"NOISE_SHAPE"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
{
|
||||
"last_node_id": 59,
|
||||
"last_link_id": 143,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 56,
|
||||
"type": "Load Motionctrl Checkpoint",
|
||||
"pos": [
|
||||
428,
|
||||
-149
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 142
|
||||
},
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MOTIONCTRL",
|
||||
"links": [
|
||||
127,
|
||||
128
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
},
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "EMBEDDER",
|
||||
"links": [
|
||||
129
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 1
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"links": [
|
||||
130
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 2
|
||||
},
|
||||
{
|
||||
"name": "ddim_sampler",
|
||||
"type": "SAMPLER",
|
||||
"links": [
|
||||
131
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 3
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Load Motionctrl Checkpoint"
|
||||
},
|
||||
"widgets_values": [
|
||||
"motionctrl.pth",
|
||||
16
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 48,
|
||||
"type": "Load Motion Camera Preset",
|
||||
"pos": [
|
||||
429,
|
||||
213
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 58
|
||||
},
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "POINTS",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
141
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Load Motion Camera Preset"
|
||||
},
|
||||
"widgets_values": [
|
||||
"U"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 49,
|
||||
"type": "Load Motion Traj Preset",
|
||||
"pos": [
|
||||
428,
|
||||
343
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 82
|
||||
},
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "POINTS",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
142
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Load Motion Traj Preset"
|
||||
},
|
||||
"widgets_values": [
|
||||
"curve_1",
|
||||
16
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 59,
|
||||
"type": "Motionctrl Sample Simple",
|
||||
"pos": [
|
||||
1384,
|
||||
-62
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 378
|
||||
},
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MOTIONCTRL",
|
||||
"link": 128
|
||||
},
|
||||
{
|
||||
"name": "clip",
|
||||
"type": "EMBEDDER",
|
||||
"link": 129
|
||||
},
|
||||
{
|
||||
"name": "vae",
|
||||
"type": "VAE",
|
||||
"link": 130
|
||||
},
|
||||
{
|
||||
"name": "ddim_sampler",
|
||||
"type": "SAMPLER",
|
||||
"link": 131
|
||||
},
|
||||
{
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"link": 134
|
||||
},
|
||||
{
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"link": 135
|
||||
},
|
||||
{
|
||||
"name": "traj_list",
|
||||
"type": "TRAJ_LIST",
|
||||
"link": 136
|
||||
},
|
||||
{
|
||||
"name": "rt_list",
|
||||
"type": "RT_LIST",
|
||||
"link": 137
|
||||
},
|
||||
{
|
||||
"name": "traj",
|
||||
"type": "TRAJ_FEATURES",
|
||||
"link": 138
|
||||
},
|
||||
{
|
||||
"name": "rt",
|
||||
"type": "RT",
|
||||
"link": 139
|
||||
},
|
||||
{
|
||||
"name": "noise_shape",
|
||||
"type": "NOISE_SHAPE",
|
||||
"link": 140
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
143
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Motionctrl Sample Simple"
|
||||
},
|
||||
"widgets_values": [
|
||||
12,
|
||||
1028,
|
||||
"randomize",
|
||||
"https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html",
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 45,
|
||||
"type": "PreviewImage",
|
||||
"pos": [
|
||||
1852,
|
||||
62
|
||||
],
|
||||
"size": [
|
||||
210,
|
||||
246
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 143
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewImage"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 58,
|
||||
"type": "Motionctrl Cond",
|
||||
"pos": [
|
||||
869,
|
||||
100
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
320
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "model",
|
||||
"type": "MOTIONCTRL",
|
||||
"link": 127
|
||||
},
|
||||
{
|
||||
"name": "camera",
|
||||
"type": "STRING",
|
||||
"link": 141,
|
||||
"widget": {
|
||||
"name": "camera"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "traj",
|
||||
"type": "STRING",
|
||||
"link": 142,
|
||||
"widget": {
|
||||
"name": "traj"
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "positive",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
134
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
},
|
||||
{
|
||||
"name": "negative",
|
||||
"type": "CONDITIONING",
|
||||
"links": [
|
||||
135
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 1
|
||||
},
|
||||
{
|
||||
"name": "traj_list",
|
||||
"type": "TRAJ_LIST",
|
||||
"links": [
|
||||
136
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 2
|
||||
},
|
||||
{
|
||||
"name": "rt_list",
|
||||
"type": "RT_LIST",
|
||||
"links": [
|
||||
137
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 3
|
||||
},
|
||||
{
|
||||
"name": "traj",
|
||||
"type": "TRAJ_FEATURES",
|
||||
"links": [
|
||||
138
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 4
|
||||
},
|
||||
{
|
||||
"name": "rt",
|
||||
"type": "RT",
|
||||
"links": [
|
||||
139
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 5
|
||||
},
|
||||
{
|
||||
"name": "noise_shape",
|
||||
"type": "NOISE_SHAPE",
|
||||
"links": [
|
||||
140
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 6
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Motionctrl Cond"
|
||||
},
|
||||
"widgets_values": [
|
||||
"a rose swaying in the wind",
|
||||
"[[1,0,0,0,0,1,0,0,0,0,1,0.2]]",
|
||||
"[[117, 102]]",
|
||||
"control both camera and object motion"
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
127,
|
||||
56,
|
||||
0,
|
||||
58,
|
||||
0,
|
||||
"MOTIONCTRL"
|
||||
],
|
||||
[
|
||||
128,
|
||||
56,
|
||||
0,
|
||||
59,
|
||||
0,
|
||||
"MOTIONCTRL"
|
||||
],
|
||||
[
|
||||
129,
|
||||
56,
|
||||
1,
|
||||
59,
|
||||
1,
|
||||
"EMBEDDER"
|
||||
],
|
||||
[
|
||||
130,
|
||||
56,
|
||||
2,
|
||||
59,
|
||||
2,
|
||||
"VAE"
|
||||
],
|
||||
[
|
||||
131,
|
||||
56,
|
||||
3,
|
||||
59,
|
||||
3,
|
||||
"SAMPLER"
|
||||
],
|
||||
[
|
||||
134,
|
||||
58,
|
||||
0,
|
||||
59,
|
||||
4,
|
||||
"CONDITIONING"
|
||||
],
|
||||
[
|
||||
135,
|
||||
58,
|
||||
1,
|
||||
59,
|
||||
5,
|
||||
"CONDITIONING"
|
||||
],
|
||||
[
|
||||
136,
|
||||
58,
|
||||
2,
|
||||
59,
|
||||
6,
|
||||
"TRAJ_LIST"
|
||||
],
|
||||
[
|
||||
137,
|
||||
58,
|
||||
3,
|
||||
59,
|
||||
7,
|
||||
"RT_LIST"
|
||||
],
|
||||
[
|
||||
138,
|
||||
58,
|
||||
4,
|
||||
59,
|
||||
8,
|
||||
"TRAJ_FEATURES"
|
||||
],
|
||||
[
|
||||
139,
|
||||
58,
|
||||
5,
|
||||
59,
|
||||
9,
|
||||
"RT"
|
||||
],
|
||||
[
|
||||
140,
|
||||
58,
|
||||
6,
|
||||
59,
|
||||
10,
|
||||
"NOISE_SHAPE"
|
||||
],
|
||||
[
|
||||
141,
|
||||
48,
|
||||
0,
|
||||
58,
|
||||
1,
|
||||
"STRING"
|
||||
],
|
||||
[
|
||||
142,
|
||||
49,
|
||||
0,
|
||||
58,
|
||||
2,
|
||||
"STRING"
|
||||
],
|
||||
[
|
||||
143,
|
||||
59,
|
||||
0,
|
||||
45,
|
||||
0,
|
||||
"IMAGE"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -1,250 +0,0 @@
|
||||
{
|
||||
"last_node_id": 10,
|
||||
"last_link_id": 18,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 7,
|
||||
"type": "VHS_VideoCombine",
|
||||
"pos": [
|
||||
1244,
|
||||
513
|
||||
],
|
||||
"size": [
|
||||
315,
|
||||
539
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 11
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "VHS_VideoCombine"
|
||||
},
|
||||
"widgets_values": {
|
||||
"frame_rate": 8,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "AnimateDiff",
|
||||
"format": "video/h264-mp4",
|
||||
"pingpong": false,
|
||||
"save_image": true,
|
||||
"crf": 20,
|
||||
"save_metadata": true,
|
||||
"audio_file": "",
|
||||
"videopreview": {
|
||||
"hidden": false,
|
||||
"paused": false,
|
||||
"params": {
|
||||
"filename": "AnimateDiff_00449.mp4",
|
||||
"subfolder": "",
|
||||
"type": "output",
|
||||
"format": "video/h264-mp4"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "PreviewImage",
|
||||
"pos": [
|
||||
1324,
|
||||
200
|
||||
],
|
||||
"size": {
|
||||
"0": 210,
|
||||
"1": 246
|
||||
},
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "images",
|
||||
"type": "IMAGE",
|
||||
"link": 10
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewImage"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "Load Motion Traj Preset",
|
||||
"pos": [
|
||||
213,
|
||||
368
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 82
|
||||
},
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "POINTS",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
17
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Load Motion Traj Preset"
|
||||
},
|
||||
"widgets_values": [
|
||||
"shaking_10",
|
||||
32
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 1,
|
||||
"type": "Load Motion Camera Preset",
|
||||
"pos": [
|
||||
221,
|
||||
126
|
||||
],
|
||||
"size": {
|
||||
"0": 315,
|
||||
"1": 58
|
||||
},
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"outputs": [
|
||||
{
|
||||
"name": "POINTS",
|
||||
"type": "STRING",
|
||||
"links": [
|
||||
14,
|
||||
18
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Load Motion Camera Preset"
|
||||
},
|
||||
"widgets_values": [
|
||||
"R"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "Motionctrl Sample",
|
||||
"pos": [
|
||||
778,
|
||||
199
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
344
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "camera",
|
||||
"type": "STRING",
|
||||
"link": 18,
|
||||
"widget": {
|
||||
"name": "camera"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "traj",
|
||||
"type": "STRING",
|
||||
"link": 17,
|
||||
"widget": {
|
||||
"name": "traj"
|
||||
}
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": [
|
||||
10,
|
||||
11
|
||||
],
|
||||
"shape": 3,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Motionctrl Sample"
|
||||
},
|
||||
"widgets_values": [
|
||||
"1dog walking",
|
||||
"[[1,0,0,0,0,1,0,0,0,0,1,0],[1,0,0,0,0,1,0,1.25550248777405,0,0,1,0],[1,0,0,0,0,1,0,1.9272470339778311,0,0,1,0],[1,0,0,1.7694104203949301,0,1,0,1.9272470339778311,0,0,1,0]]",
|
||||
"[[512,512]]",
|
||||
16,
|
||||
50,
|
||||
1234,
|
||||
"fixed",
|
||||
"https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html",
|
||||
true,
|
||||
true
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
10,
|
||||
6,
|
||||
0,
|
||||
4,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
11,
|
||||
6,
|
||||
0,
|
||||
7,
|
||||
0,
|
||||
"IMAGE"
|
||||
],
|
||||
[
|
||||
14,
|
||||
1,
|
||||
0,
|
||||
10,
|
||||
0,
|
||||
"*"
|
||||
],
|
||||
[
|
||||
17,
|
||||
2,
|
||||
0,
|
||||
6,
|
||||
1,
|
||||
"STRING"
|
||||
],
|
||||
[
|
||||
18,
|
||||
1,
|
||||
0,
|
||||
6,
|
||||
0,
|
||||
"STRING"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
Reference in New Issue
Block a user