From 1cbf898083dad0454db684b8a3449ae49366c85d Mon Sep 17 00:00:00 2001 From: fles Date: Sun, 21 Apr 2024 17:13:04 +0800 Subject: [PATCH] camera poses nodes --- animatediff/nodes.py | 10 +- animatediff/nodes_cameractrl.py | 159 ++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 1 deletion(-) diff --git a/animatediff/nodes.py b/animatediff/nodes.py index 674f779..49d06f7 100644 --- a/animatediff/nodes.py +++ b/animatediff/nodes.py @@ -6,7 +6,7 @@ from .nodes_gen1 import (AnimateDiffLoaderGen1, LegacyAnimateDiffLoaderWithConte from .nodes_gen2 import (UseEvolvedSamplingNode, ApplyAnimateDiffModelNode, ApplyAnimateDiffModelBasicNode, ADKeyframeNode, LoadAnimateDiffModelNode) from .nodes_animatelcmi2v import (ApplyAnimateLCMI2VModel, LoadAnimateLCMI2VModelNode, LoadAnimateDiffAndInjectI2VNode, UpscaleAndVaeEncode) -from .nodes_cameractrl import (LoadAnimateDiffModelWithCameraCtrl, ApplyAnimateDiffWithCameraCtrl, LoadCameraPoses) +from .nodes_cameractrl import (LoadAnimateDiffModelWithCameraCtrl, ApplyAnimateDiffWithCameraCtrl, LoadCameraPoses, CameraPoseBasic, CameraPoseJoin, CameraPoseCombine, CameraCtrlPose) from .nodes_multival import MultivalDynamicNode, MultivalScaledMaskNode from .nodes_sample import (FreeInitOptionsNode, NoiseLayerAddWeightedNode, SampleSettingsNode, NoiseLayerAddNode, NoiseLayerReplaceNode, IterationOptionsNode, CustomCFGNode, CustomCFGKeyframeNode) @@ -98,6 +98,10 @@ NODE_CLASS_MAPPINGS = { "ADE_ApplyAnimateDiffModelWithCameraCtrl": ApplyAnimateDiffWithCameraCtrl, "ADE_LoadAnimateDiffModelWithCameraCtrl": LoadAnimateDiffModelWithCameraCtrl, "ADE_LoadCameraPoses": LoadCameraPoses, + "ADE_CameraPoseBasic": CameraPoseBasic, + "ADE_CameraPoseJoin": CameraPoseJoin, + "ADE_CameraPoseCombine": CameraPoseCombine, + "ADE_CameraCtrlPose": CameraCtrlPose, # MaskedLoraLoader #"ADE_MaskedLoadLora": MaskedLoraLoader, # Deprecated Nodes @@ -175,6 +179,10 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ADE_ApplyAnimateDiffModelWithCameraCtrl": "Apply AnimateDiff+CameraCtrl Model 🎭🅐🅓②", "ADE_LoadAnimateDiffModelWithCameraCtrl": "Load AnimateDiff+CameraCtrl Model 🎭🅐🅓②", "ADE_LoadCameraPoses": "Load CameraCtrl Poses 🎭🅐🅓②", + "ADE_CameraPoseBasic": "CameraCtrl Basic Poses 🎭🅐🅓②", + "ADE_CameraPoseJoin": "CameraCtrl Join Poses 🎭🅐🅓②", + "ADE_CameraPoseCombine": "CameraCtrl Combine Poses 🎭🅐🅓②", + "ADE_CameraCtrlPose": "Generate CameraCtrl Poses 🎭🅐🅓②", # MaskedLoraLoader #"ADE_MaskedLoadLora": "Load LoRA (Masked) 🎭🅐🅓", # Deprecated Nodes diff --git a/animatediff/nodes_cameractrl.py b/animatediff/nodes_cameractrl.py index 402e496..630481f 100644 --- a/animatediff/nodes_cameractrl.py +++ b/animatediff/nodes_cameractrl.py @@ -2,6 +2,9 @@ from typing import Union import os import folder_paths +import copy +import json +import numpy as np from .ad_settings import AnimateDiffSettings from .adapter_cameractrl import CameraEntry @@ -12,6 +15,67 @@ from .motion_lora import MotionLoraList from .model_injection import (MotionModelGroup, MotionModelPatcher, load_motion_module_gen2, inject_camera_encoder_into_model) from .nodes_gen2 import ApplyAnimateDiffModelNode +CAMERA = { + # T + "base_T_norm": 1.5, + "base_angle": np.pi/3, + + "Static": { "angle":[0., 0., 0.], "T":[0., 0., 0.]}, + "Pan Up": { "angle":[0., 0., 0.], "T":[0., 1., 0.]}, + "Pan Down": { "angle":[0., 0., 0.], "T":[0.,-1.,0.]}, + "Pan Left": { "angle":[0., 0., 0.], "T":[1.,0.,0.]}, + "Pan Right": { "angle":[0., 0., 0.], "T": [-1.,0.,0.]}, + "Zoom In": { "angle":[0., 0., 0.], "T": [0.,0.,-2.]}, + "Zoom Out": { "angle":[0., 0., 0.], "T": [0.,0.,2.]}, + "ACW": { "angle": [0., 0., 1.], "T":[0., 0., 0.]}, + "CW": { "angle": [0., 0., -1.], "T":[0., 0., 0.]}, +} + +def compute_R_form_rad_angle(angles): + theta_x, theta_y, theta_z = angles + Rx = np.array([[1, 0, 0], + [0, np.cos(theta_x), -np.sin(theta_x)], + [0, np.sin(theta_x), np.cos(theta_x)]]) + + Ry = np.array([[np.cos(theta_y), 0, np.sin(theta_y)], + [0, 1, 0], + [-np.sin(theta_y), 0, np.cos(theta_y)]]) + + Rz = np.array([[np.cos(theta_z), -np.sin(theta_z), 0], + [np.sin(theta_z), np.cos(theta_z), 0], + [0, 0, 1]]) + + R = np.dot(Rz, np.dot(Ry, Rx)) + return R + +def get_camera_motion(angle, T, speed, n=16): + RT = [] + for i in range(n): + _angle = (i/n)*speed*(CAMERA["base_angle"])*angle + R = compute_R_form_rad_angle(_angle) + # _T = (i/n)*speed*(T.reshape(3,1)) + _T=(i/n)*speed*(CAMERA["base_T_norm"])*(T.reshape(3,1)) + _RT = np.concatenate([R,_T], axis=1) + RT.append(_RT) + RT = np.stack(RT) + return RT + +def combine_camera_motion(RT_0, RT_1): + RT = copy.deepcopy(RT_0[-1]) + R = RT[:,:3] + R_inv = RT[:,:3].T + T = RT[:,-1] + + temp = [] + for _RT in RT_1: + _RT[:,:3] = np.dot(_RT[:,:3], R) + _RT[:,-1] = _RT[:,-1] + np.dot(np.dot(_RT[:,:3], R_inv), T) + temp.append(_RT) + + RT_1 = np.stack(temp) + + return np.concatenate([RT_0, RT_1], axis=0) + class ApplyAnimateDiffWithCameraCtrl: @classmethod @@ -102,3 +166,98 @@ class LoadCameraPoses: poses = [pose.strip().split(' ') for pose in poses[1:]] poses = [[float(x) for x in pose] for pose in poses] return (poses,) + +class CameraPoseBasic: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_pose":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "speed":("FLOAT",{"default":1.0}), + "video_length":("INT",{"default":16}), + }, + } + + RETURN_TYPES = ("CameraPose",) + FUNCTION = "camera_pose_basic" + CATEGORY = "Animate Diff 🎭🅐🅓/② Gen2 nodes ②/CameraCtrl" + + def camera_pose_basic(self,camera_pose,speed,video_length): + motion_list = [camera_pose] + mode = "Basic Camera Poses" + angle = np.array(CAMERA[motion_list[0]]["angle"]) + T = np.array(CAMERA[motion_list[0]]["T"]) + RT = get_camera_motion(angle, T, speed, video_length) + return (RT,) + +class CameraPoseJoin: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_pose1":("CameraPose",), + "camera_pose2":("CameraPose",), + }, + } + + RETURN_TYPES = ("CameraPose",) + FUNCTION = "camera_pose_join" + CATEGORY = "Animate Diff 🎭🅐🅓/② Gen2 nodes ②/CameraCtrl" + + def camera_pose_join(self,camera_pose1,camera_pose2): + RT = combine_camera_motion(camera_pose1, camera_pose2) + return (RT,) + +class CameraPoseCombine: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_pose1":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "camera_pose2":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "camera_pose3":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "camera_pose4":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "speed":("FLOAT",{"default":1.0}), + "video_length":("INT",{"default":16}), + }, + } + + RETURN_TYPES = ("CameraPose",) + FUNCTION = "camera_pose_combine" + CATEGORY = "Animate Diff 🎭🅐🅓/② Gen2 nodes ②/CameraCtrl" + + def camera_pose_combine(self,camera_pose1,camera_pose2,camera_pose3,camera_pose4,speed,video_length): + angle = np.array(CAMERA[camera_pose1]["angle"]) + np.array(CAMERA[camera_pose2]["angle"]) + np.array(CAMERA[camera_pose3]["angle"]) + np.array(CAMERA[camera_pose4]["angle"]) + T = np.array(CAMERA[camera_pose1]["T"]) + np.array(CAMERA[camera_pose2]["T"]) + np.array(CAMERA[camera_pose3]["T"]) + np.array(CAMERA[camera_pose4]["T"]) + RT = get_camera_motion(angle, T, speed, video_length) + return (RT,) + +class CameraCtrlPose: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_pose":("CameraPose",), + "fx":("FLOAT",{"default":0.474812461, "min": 0, "max": 1, "step": 0.000000001}), + "fy":("FLOAT",{"default":0.844111024, "min": 0, "max": 1, "step": 0.000000001}), + "cx":("FLOAT",{"default":0.5, "min": 0, "max": 1, "step": 0.01}), + "cy":("FLOAT",{"default":0.5, "min": 0, "max": 1, "step": 0.01}), + }, + } + + RETURN_TYPES = ("CAMERACTRL_POSES","INT",) + RETURN_NAMES = ("cameractrl_poses","video_length",) + FUNCTION = "camera_ctrl_pose" + CATEGORY = "Animate Diff 🎭🅐🅓/② Gen2 nodes ②/CameraCtrl" + + def camera_ctrl_pose(self,camera_pose,fx,fy,cx,cy): + camera_pose_list=camera_pose.tolist() + trajs=[] + for cp in camera_pose_list: + traj=[0,fx,fy,cx,cy,0,0] + traj.extend(cp[0]) + traj.extend(cp[1]) + traj.extend(cp[2]) + trajs.append(traj) + + return (trajs,len(trajs),) \ No newline at end of file