nodes/Kosinkadink-ComfyUI-AnimateDiff-Evolved
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

camera poses nodes

Browse Source
This commit is contained in:
fles
2024-04-21 17:13:04 +08:00
parent bda8b6aad8
commit 1cbf898083
2 changed files with 168 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
animatediff/nodes.py
+9 -1
View File
@@ -6,7 +6,7 @@ from .nodes_gen1 import (AnimateDiffLoaderGen1, LegacyAnimateDiffLoaderWithConte
from .nodes_gen2 import (UseEvolvedSamplingNode, ApplyAnimateDiffModelNode, ApplyAnimateDiffModelBasicNode, ADKeyframeNode,
LoadAnimateDiffModelNode)
from .nodes_animatelcmi2v import (ApplyAnimateLCMI2VModel, LoadAnimateLCMI2VModelNode, LoadAnimateDiffAndInjectI2VNode, UpscaleAndVaeEncode)
from .nodes_cameractrl import (LoadAnimateDiffModelWithCameraCtrl, ApplyAnimateDiffWithCameraCtrl, LoadCameraPoses)
from .nodes_cameractrl import (LoadAnimateDiffModelWithCameraCtrl, ApplyAnimateDiffWithCameraCtrl, LoadCameraPoses, CameraPoseBasic, CameraPoseJoin, CameraPoseCombine, CameraCtrlPose)
from .nodes_multival import MultivalDynamicNode, MultivalScaledMaskNode
from .nodes_sample import (FreeInitOptionsNode, NoiseLayerAddWeightedNode, SampleSettingsNode, NoiseLayerAddNode, NoiseLayerReplaceNode, IterationOptionsNode,
CustomCFGNode, CustomCFGKeyframeNode)
@@ -98,6 +98,10 @@ NODE_CLASS_MAPPINGS = {
"ADE_ApplyAnimateDiffModelWithCameraCtrl": ApplyAnimateDiffWithCameraCtrl,
"ADE_LoadAnimateDiffModelWithCameraCtrl": LoadAnimateDiffModelWithCameraCtrl,
"ADE_LoadCameraPoses": LoadCameraPoses,
"ADE_CameraPoseBasic": CameraPoseBasic,
"ADE_CameraPoseJoin": CameraPoseJoin,
"ADE_CameraPoseCombine": CameraPoseCombine,
"ADE_CameraCtrlPose": CameraCtrlPose,
# MaskedLoraLoader
#"ADE_MaskedLoadLora": MaskedLoraLoader,
# Deprecated Nodes
@@ -175,6 +179,10 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"ADE_ApplyAnimateDiffModelWithCameraCtrl": "Apply AnimateDiff+CameraCtrl Model 🎭🅐🅓②",
"ADE_LoadAnimateDiffModelWithCameraCtrl": "Load AnimateDiff+CameraCtrl Model 🎭🅐🅓②",
"ADE_LoadCameraPoses": "Load CameraCtrl Poses 🎭🅐🅓②",
"ADE_CameraPoseBasic": "CameraCtrl Basic Poses 🎭🅐🅓②",
"ADE_CameraPoseJoin": "CameraCtrl Join Poses 🎭🅐🅓②",
"ADE_CameraPoseCombine": "CameraCtrl Combine Poses 🎭🅐🅓②",
"ADE_CameraCtrlPose": "Generate CameraCtrl Poses 🎭🅐🅓②",
# MaskedLoraLoader
#"ADE_MaskedLoadLora": "Load LoRA (Masked) 🎭🅐🅓",
# Deprecated Nodes
animatediff/nodes_cameractrl.py
+159
View File
@@ -2,6 +2,9 @@ from typing import Union
import os
import folder_paths
import copy
import json
import numpy as np
from .ad_settings import AnimateDiffSettings
from .adapter_cameractrl import CameraEntry
@@ -12,6 +15,67 @@ from .motion_lora import MotionLoraList
from .model_injection import (MotionModelGroup, MotionModelPatcher, load_motion_module_gen2, inject_camera_encoder_into_model)
from .nodes_gen2 import ApplyAnimateDiffModelNode
CAMERA = {
# T
"base_T_norm": 1.5,
"base_angle": np.pi/3,
"Static": { "angle":[0., 0., 0.], "T":[0., 0., 0.]},
"Pan Up": { "angle":[0., 0., 0.], "T":[0., 1., 0.]},
"Pan Down": { "angle":[0., 0., 0.], "T":[0.,-1.,0.]},
"Pan Left": { "angle":[0., 0., 0.], "T":[1.,0.,0.]},
"Pan Right": { "angle":[0., 0., 0.], "T": [-1.,0.,0.]},
"Zoom In": { "angle":[0., 0., 0.], "T": [0.,0.,-2.]},
"Zoom Out": { "angle":[0., 0., 0.], "T": [0.,0.,2.]},
"ACW": { "angle": [0., 0., 1.], "T":[0., 0., 0.]},
"CW": { "angle": [0., 0., -1.], "T":[0., 0., 0.]},
}
def compute_R_form_rad_angle(angles):
theta_x, theta_y, theta_z = angles
Rx = np.array([[1, 0, 0],
[0, np.cos(theta_x), -np.sin(theta_x)],
[0, np.sin(theta_x), np.cos(theta_x)]])
Ry = np.array([[np.cos(theta_y), 0, np.sin(theta_y)],
[0, 1, 0],
[-np.sin(theta_y), 0, np.cos(theta_y)]])
Rz = np.array([[np.cos(theta_z), -np.sin(theta_z), 0],
[np.sin(theta_z), np.cos(theta_z), 0],
[0, 0, 1]])
R = np.dot(Rz, np.dot(Ry, Rx))
return R
def get_camera_motion(angle, T, speed, n=16):
RT = []
for i in range(n):
_angle = (i/n)*speed*(CAMERA["base_angle"])*angle
R = compute_R_form_rad_angle(_angle)
# _T = (i/n)*speed*(T.reshape(3,1))
_T=(i/n)*speed*(CAMERA["base_T_norm"])*(T.reshape(3,1))
_RT = np.concatenate([R,_T], axis=1)
RT.append(_RT)
RT = np.stack(RT)
return RT
def combine_camera_motion(RT_0, RT_1):
RT = copy.deepcopy(RT_0[-1])
R = RT[:,:3]
R_inv = RT[:,:3].T
T = RT[:,-1]
temp = []
for _RT in RT_1:
_RT[:,:3] = np.dot(_RT[:,:3], R)
_RT[:,-1] = _RT[:,-1] + np.dot(np.dot(_RT[:,:3], R_inv), T)
temp.append(_RT)
RT_1 = np.stack(temp)
return np.concatenate([RT_0, RT_1], axis=0)
class ApplyAnimateDiffWithCameraCtrl:
@classmethod
@@ -102,3 +166,98 @@ class LoadCameraPoses:
poses = [pose.strip().split(' ') for pose in poses[1:]]
poses = [[float(x) for x in pose] for pose in poses]
return (poses,)
class CameraPoseBasic:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"camera_pose":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}),
"speed":("FLOAT",{"default":1.0}),
"video_length":("INT",{"default":16}),
},
}
RETURN_TYPES = ("CameraPose",)
FUNCTION = "camera_pose_basic"
CATEGORY = "Animate Diff 🎭🅐🅓/② Gen2 nodes ②/CameraCtrl"
def camera_pose_basic(self,camera_pose,speed,video_length):
motion_list = [camera_pose]
mode = "Basic Camera Poses"
angle = np.array(CAMERA[motion_list[0]]["angle"])
T = np.array(CAMERA[motion_list[0]]["T"])
RT = get_camera_motion(angle, T, speed, video_length)
return (RT,)
class CameraPoseJoin:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"camera_pose1":("CameraPose",),
"camera_pose2":("CameraPose",),
},
}
RETURN_TYPES = ("CameraPose",)
FUNCTION = "camera_pose_join"
CATEGORY = "Animate Diff 🎭🅐🅓/② Gen2 nodes ②/CameraCtrl"
def camera_pose_join(self,camera_pose1,camera_pose2):
RT = combine_camera_motion(camera_pose1, camera_pose2)
return (RT,)
class CameraPoseCombine:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"camera_pose1":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}),
"camera_pose2":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}),
"camera_pose3":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}),
"camera_pose4":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}),
"speed":("FLOAT",{"default":1.0}),
"video_length":("INT",{"default":16}),
},
}
RETURN_TYPES = ("CameraPose",)
FUNCTION = "camera_pose_combine"
CATEGORY = "Animate Diff 🎭🅐🅓/② Gen2 nodes ②/CameraCtrl"
def camera_pose_combine(self,camera_pose1,camera_pose2,camera_pose3,camera_pose4,speed,video_length):
angle = np.array(CAMERA[camera_pose1]["angle"]) + np.array(CAMERA[camera_pose2]["angle"]) + np.array(CAMERA[camera_pose3]["angle"]) + np.array(CAMERA[camera_pose4]["angle"])
T = np.array(CAMERA[camera_pose1]["T"]) + np.array(CAMERA[camera_pose2]["T"]) + np.array(CAMERA[camera_pose3]["T"]) + np.array(CAMERA[camera_pose4]["T"])
RT = get_camera_motion(angle, T, speed, video_length)
return (RT,)
class CameraCtrlPose:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"camera_pose":("CameraPose",),
"fx":("FLOAT",{"default":0.474812461, "min": 0, "max": 1, "step": 0.000000001}),
"fy":("FLOAT",{"default":0.844111024, "min": 0, "max": 1, "step": 0.000000001}),
"cx":("FLOAT",{"default":0.5, "min": 0, "max": 1, "step": 0.01}),
"cy":("FLOAT",{"default":0.5, "min": 0, "max": 1, "step": 0.01}),
},
}
RETURN_TYPES = ("CAMERACTRL_POSES","INT",)
RETURN_NAMES = ("cameractrl_poses","video_length",)
FUNCTION = "camera_ctrl_pose"
CATEGORY = "Animate Diff 🎭🅐🅓/② Gen2 nodes ②/CameraCtrl"
def camera_ctrl_pose(self,camera_pose,fx,fy,cx,cy):
camera_pose_list=camera_pose.tolist()
trajs=[]
for cp in camera_pose_list:
traj=[0,fx,fy,cx,cy,0,0]
traj.extend(cp[0])
traj.extend(cp[1])
traj.extend(cp[2])
trajs.append(traj)
return (trajs,len(trajs),)