68 lines
2.5 KiB
Plaintext
68 lines
2.5 KiB
Plaintext
class Load_CLIP_Vision_With_Projection:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"pretrained_clip_vision_folder_path": ("STRING", {"default": "./pretrained_weights/image_encoder/", "multiline": False}),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = (
|
|
"CLIP_VISION",
|
|
)
|
|
RETURN_NAMES = (
|
|
"clip_vision",
|
|
)
|
|
FUNCTION = "load_clip_vision"
|
|
|
|
CATEGORY = "AnimateAnyone-Evolved/loaders"
|
|
|
|
def load_clip_vision(self, pretrained_clip_vision_folder_path):
|
|
if not os.path.isabs(pretrained_clip_vision_folder_path):
|
|
pretrained_clip_vision_folder_path = os.path.join(ROOT_PATH, pretrained_clip_vision_folder_path)
|
|
|
|
clip_vision = CLIPVisionModelWithProjection.from_pretrained(
|
|
pretrained_clip_vision_folder_path
|
|
).to(dtype=WEIGHT_DETYPE, device=DEVICE)
|
|
|
|
return (clip_vision,)
|
|
|
|
class CLIP_Vision_With_Projection_Encode:
|
|
@classmethod
|
|
def INPUT_TYPES(s):
|
|
return {
|
|
"required": {
|
|
"clip_vision": ("CLIP_VISION",),
|
|
"reference_image": ("IMAGE",),
|
|
},
|
|
}
|
|
|
|
RETURN_TYPES = (
|
|
"CLIP_VISION_OUTPUT",
|
|
)
|
|
RETURN_NAMES = (
|
|
"clip_vision_output",
|
|
)
|
|
FUNCTION = "clip_vision_encode"
|
|
|
|
CATEGORY = "AnimateAnyone-Evolved/processors"
|
|
OUTPUT_NODE = True
|
|
def clip_vision_encode(self, clip_vision, reference_image):
|
|
clip_image_processor = CLIPImageProcessor()
|
|
|
|
clip_image = clip_image_processor.preprocess(
|
|
reference_image, do_resize=True, size={"height": CONFIG.clip_img_height, "width": CONFIG.clip_img_width}, return_tensors="pt"
|
|
).pixel_values
|
|
clip_image_embeds = clip_vision(
|
|
clip_image.to(DEVICE, dtype=clip_vision.dtype)
|
|
).image_embeds
|
|
encoder_hidden_states = clip_image_embeds.unsqueeze(1)
|
|
#print(f"encoder_hidden_states.shape: {encoder_hidden_states.shape}\n clip_image_embeds.shape: {clip_image_embeds.shape}")
|
|
#encoder_hidden_states.shape: torch.Size([1, 1, 768]) clip_image_embeds.shape: torch.Size([1, 768])
|
|
uncond_encoder_hidden_states = torch.zeros_like(encoder_hidden_states)
|
|
|
|
with open("_Test_CLIP_VISION_Projected_OUTPUT.txt", "w") as file1:
|
|
# Writing data to a file
|
|
file1.write(f"[CLIP_VISION_Projected_OUTPUT]: \nclip_image_embeds shape: {clip_image_embeds.shape}\nclip_image_embeds: {clip_image_embeds}")
|
|
|
|
return (clip_image_embeds,) |