Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d312c62982 |
@@ -81,7 +81,7 @@ class DownloadAndLoadDynamiCrafterModel:
|
|||||||
'ckpt_name': model,
|
'ckpt_name': model,
|
||||||
'fp8_unet': fp8_unet
|
'fp8_unet': fp8_unet
|
||||||
}
|
}
|
||||||
if not hasattr(self, 'model') or self.model == None or custom_config != self.current_config:
|
if not hasattr(self, 'model') or self.model is None or custom_config != self.current_config:
|
||||||
self.current_config = custom_config
|
self.current_config = custom_config
|
||||||
download_path = os.path.join(folder_paths.models_dir, "checkpoints", "dynamicrafter")
|
download_path = os.path.join(folder_paths.models_dir, "checkpoints", "dynamicrafter")
|
||||||
model_path = os.path.join(download_path, model)
|
model_path = os.path.join(download_path, model)
|
||||||
@@ -557,8 +557,6 @@ class DynamiCrafterI2V:
|
|||||||
text_emb = positive[0][0].to(device)
|
text_emb = positive[0][0].to(device)
|
||||||
|
|
||||||
cond_images = clip_vision.encode_image(image.permute(0, 2, 3, 1))['last_hidden_state'].to(device)
|
cond_images = clip_vision.encode_image(image.permute(0, 2, 3, 1))['last_hidden_state'].to(device)
|
||||||
cond_images = torch.sum(cond_images, dim=0).unsqueeze(0)
|
|
||||||
cond_images = torch.mean(cond_images, dim=0).unsqueeze(0)
|
|
||||||
|
|
||||||
img_emb = self.model.image_proj_model(cond_images)
|
img_emb = self.model.image_proj_model(cond_images)
|
||||||
|
|
||||||
@@ -816,11 +814,10 @@ class ToonCrafterInterpolation:
|
|||||||
pbar = comfy.utils.ProgressBar(len(images) - 1)
|
pbar = comfy.utils.ProgressBar(len(images) - 1)
|
||||||
autocast_condition = (dtype != torch.float32) and not comfy.model_management.is_device_mps(device)
|
autocast_condition = (dtype != torch.float32) and not comfy.model_management.is_device_mps(device)
|
||||||
with torch.autocast(comfy.model_management.get_autocast_device(device), dtype=dtype) if autocast_condition else nullcontext():
|
with torch.autocast(comfy.model_management.get_autocast_device(device), dtype=dtype) if autocast_condition else nullcontext():
|
||||||
for i in range(len(images) - 1) if len(images) > 1 else range(len(images)):
|
for i in range(len(images) - 1):
|
||||||
videos, videos2 = None, None
|
videos, videos2 = None, None
|
||||||
mm.soft_empty_cache()
|
mm.soft_empty_cache()
|
||||||
image = images[i].unsqueeze(0)
|
image = images[i].unsqueeze(0)
|
||||||
if len(images) !=1:
|
|
||||||
image2 = images[i+1].unsqueeze(0)
|
image2 = images[i+1].unsqueeze(0)
|
||||||
|
|
||||||
B, C, H, W = image.shape
|
B, C, H, W = image.shape
|
||||||
@@ -834,14 +831,10 @@ class ToonCrafterInterpolation:
|
|||||||
|
|
||||||
encode_pixels = image.unsqueeze(2) * 2 - 1
|
encode_pixels = image.unsqueeze(2) * 2 - 1
|
||||||
videos = encode_pixels # bc1hw
|
videos = encode_pixels # bc1hw
|
||||||
videos = repeat(videos, 'b c t h w -> b c (repeat t) h w', repeat=frames // 2)
|
videos = repeat(videos, 'b c t h w -> b c (repeat t) h w', repeat=frames//2)
|
||||||
|
|
||||||
if len(images) == 1:
|
|
||||||
videos = torch.cat([videos, videos], dim=2)
|
|
||||||
else:
|
|
||||||
encode_pixels = image2.unsqueeze(2) * 2 - 1
|
encode_pixels = image2.unsqueeze(2) * 2 - 1
|
||||||
videos2 = encode_pixels # bc1hw
|
videos2 = encode_pixels # bc1hw
|
||||||
videos2 = repeat(videos2, 'b c t h w -> b c (repeat t) h w', repeat=frames // 2)
|
videos2 = repeat(videos2, 'b c t h w -> b c (repeat t) h w', repeat=frames//2)
|
||||||
videos = torch.cat([videos, videos2], dim=2)
|
videos = torch.cat([videos, videos2], dim=2)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -854,25 +847,23 @@ class ToonCrafterInterpolation:
|
|||||||
|
|
||||||
img_tensor_repeat = torch.zeros_like(z)
|
img_tensor_repeat = torch.zeros_like(z)
|
||||||
img_tensor_repeat[:,:,:1,:,:] = z[:,:,:1,:,:]
|
img_tensor_repeat[:,:,:1,:,:] = z[:,:,:1,:,:]
|
||||||
if len(images) !=1:
|
|
||||||
img_tensor_repeat[:,:,-1:,:,:] = z[:,:,-1:,:,:]
|
img_tensor_repeat[:,:,-1:,:,:] = z[:,:,-1:,:,:]
|
||||||
|
|
||||||
self.model.first_stage_model.to(offload_device)
|
self.model.first_stage_model.to(offload_device)
|
||||||
|
|
||||||
text_emb = positive[0][0].to(device)
|
text_emb = positive[0][0].to(device)
|
||||||
|
|
||||||
self.model.image_proj_model.to(device)
|
|
||||||
cond_images = clip_vision.encode_image(image.permute(0, 2, 3, 1))["last_hidden_state"].to(device)
|
cond_images = clip_vision.encode_image(image.permute(0, 2, 3, 1))["last_hidden_state"].to(device)
|
||||||
img_emb = self.model.image_proj_model(cond_images)
|
|
||||||
if len(images) !=1:
|
|
||||||
cond_images2 = clip_vision.encode_image(image2.permute(0, 2, 3, 1))["last_hidden_state"].to(device)
|
cond_images2 = clip_vision.encode_image(image2.permute(0, 2, 3, 1))["last_hidden_state"].to(device)
|
||||||
|
|
||||||
|
self.model.image_proj_model.to(device)
|
||||||
|
|
||||||
|
img_emb = self.model.image_proj_model(cond_images)
|
||||||
img_emb2 = self.model.image_proj_model(cond_images2)
|
img_emb2 = self.model.image_proj_model(cond_images2)
|
||||||
img_embeds = img_emb * image_embed_ratio + img_emb2 * (1.0 - image_embed_ratio)
|
img_embeds = img_emb * image_embed_ratio + img_emb2 * (1.0 - image_embed_ratio)
|
||||||
else:
|
|
||||||
img_embeds = img_emb
|
|
||||||
|
|
||||||
imtext_cond = torch.cat([text_emb, img_embeds], dim=1)
|
imtext_cond = torch.cat([text_emb, img_embeds], dim=1)
|
||||||
del cond_images, img_emb, text_emb
|
del cond_images, img_emb, img_emb2, text_emb
|
||||||
|
|
||||||
if comfy.model_management.is_device_mps(device):
|
if comfy.model_management.is_device_mps(device):
|
||||||
fs = torch.tensor([fs], dtype=torch.float32, device=self.model.device)
|
fs = torch.tensor([fs], dtype=torch.float32, device=self.model.device)
|
||||||
|
|||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "comfyui-dynamicrafterwrapper"
|
name = "comfyui-dynamicrafterwrapper"
|
||||||
description = "Wrapper nodes to use Dynami/ToonCrafter image2video and frame interpolation models in ComfyUI"
|
description = "Wrapper nodes to use Dynami/ToonCrafter image2video and frame interpolation models in ComfyUI"
|
||||||
version = "1.0.2"
|
version = "1.0.3"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
dependencies = ["einops>=0.3.0", "numpy>=1.24.2", "omegaconf>=2.1.1", "pytorch_lightning>=2.2.1", "tqdm>=4.65.0", "transformers>=4.25.1", "timm"]
|
dependencies = ["einops>=0.3.0", "numpy>=1.24.2", "omegaconf>=2.1.1", "pytorch_lightning>=2.2.1", "tqdm>=4.65.0", "transformers>=4.25.1", "timm"]
|
||||||
|
|
||||||
|
|||||||
+5
-2
@@ -34,12 +34,15 @@ def instantiate_from_config(config):
|
|||||||
|
|
||||||
|
|
||||||
def get_obj_from_str(string, reload=False):
|
def get_obj_from_str(string, reload=False):
|
||||||
package_directory_name = os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
||||||
module, cls = string.rsplit(".", 1)
|
module, cls = string.rsplit(".", 1)
|
||||||
if reload:
|
if reload:
|
||||||
module_imp = importlib.import_module(module)
|
module_imp = importlib.import_module(module)
|
||||||
importlib.reload(module_imp)
|
importlib.reload(module_imp)
|
||||||
return getattr(importlib.import_module(module, package=package_directory_name), cls)
|
try:
|
||||||
|
obj = getattr(importlib.import_module(module, package=os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), cls)
|
||||||
|
except:
|
||||||
|
obj = getattr(importlib.import_module(module, package=os.path.dirname(os.path.dirname(os.path.abspath( __file__ )))), cls)
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
def load_npz_from_dir(data_dir):
|
def load_npz_from_dir(data_dir):
|
||||||
|
|||||||
Reference in New Issue
Block a user