diff --git a/essentials.py b/essentials.py index b611e4e..20bf812 100644 --- a/essentials.py +++ b/essentials.py @@ -290,55 +290,38 @@ class ImageExpandBatch: return (out,) -# highly experimental -def identify_keyframes_pytorch(frame_differences, threshold): - keyframe_indices = [] - for i, diff in enumerate(frame_differences): - if diff >= threshold: - keyframe_indices.append(i + window_size) # Adjusting index for window size - return keyframe_indices - class ExtractKeyframes: @classmethod def INPUT_TYPES(s): return { "required": { "image": ("IMAGE",), - "window_size": ("INT", { "default": 5, "min": 1, "step": 1, }), - "max_frames": ("INT", { "default": 3, "min": 1, "step": 1, }), - #"threshold": ("FLOAT", { "default": 0.75, "min": 0.00, "max": 1.00, "step": 0.05, }), + "threshold": ("FLOAT", { "default": 0.85, "min": 0.00, "max": 1.00, "step": 0.01, }), } } - - RETURN_TYPES = ("IMAGE",) + + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("KEYFRAMES", "indexes") + FUNCTION = "execute" CATEGORY = "essentials" - def execute(self, image, window_size, max_frames): - mse_values = [] + def execute(self, image, threshold): + window_size = 2 - for i in range(window_size, image.shape[0]): - prev_frames_avg = torch.mean(image[i-window_size:i], dim=0) - current_frame = image[i] - mse = torch.mean((prev_frames_avg - current_frame) ** 2) - mse_values.append(mse.item()) + variations = torch.sum(torch.abs(image[1:] - image[:-1]), dim=[1, 2, 3]) + #variations = torch.sum((image[1:] - image[:-1]) ** 2, dim=[1, 2, 3]) + threshold = torch.quantile(variations.float(), threshold).item() - sorted_mse = sorted(mse_values, reverse=True) - if max_frames >= len(sorted_mse): - threshold = sorted_mse[-1] - else: - threshold = sorted_mse[max_frames-1] - - print(threshold) + keyframes = [] + for i in range(image.shape[0] - window_size + 1): + window = image[i:i + window_size] + variation = torch.sum(torch.abs(window[-1] - window[0])).item() - keyframe_indices = [] - for i, diff in enumerate(mse_values): - if diff >= threshold: - keyframe_indices.append(i + window_size) + if variation > threshold: + keyframes.append(i + window_size - 1) - out = image[keyframe_indices] - print(keyframe_indices) - return (out,) + return (image[keyframes], ','.join(map(str, keyframes)),) class MaskFlip: @classmethod @@ -863,7 +846,6 @@ class StableZero123_Increments: RETURN_NAMES = ("positive", "negative", "latent") FUNCTION = "encode" - CATEGORY = "essentials" def encode(self, clip_vision, init_image, vae, width, height, batch_size, elevation, azimuth, elevation_inc, azimuth_inc): @@ -882,6 +864,56 @@ class StableZero123_Increments: latent = torch.zeros([batch_size, 4, height // 8, width // 8]) return (positive, negative, {"samples":latent}) +class CLIPTextEncodeSDXLSimplified: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), + "height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), + "text": ("STRING", {"multiline": True, "default": ""}), + "clip": ("CLIP", ), + }} + RETURN_TYPES = ("CONDITIONING",) + FUNCTION = "execute" + CATEGORY = "essentials" + + def execute(self, clip, width, height, text): + crop_w = 0 + crop_h = 0 + width = width*4 + height = height*4 + target_width = width + target_height = height + text_g = text_l = text + + tokens = clip.tokenize(text_g) + tokens["l"] = clip.tokenize(text_l)["l"] + if len(tokens["l"]) != len(tokens["g"]): + empty = clip.tokenize("") + while len(tokens["l"]) < len(tokens["g"]): + tokens["l"] += empty["l"] + while len(tokens["l"]) > len(tokens["g"]): + tokens["g"] += empty["g"] + cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) + return ([[cond, {"pooled_output": pooled, "width": width, "height": height, "crop_w": crop_w, "crop_h": crop_h, "target_width": target_width, "target_height": target_height}]], ) + +class SDXLResolutionPicker: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "resolution": (["704x1408 (0.5)","704x1344 (0.52)","768x1344 (0.57)","768x1280 (0.6)","832x1216 (0.68)","832x1152 (0.72)","896x1152 (0.78)","896x1088 (0.82)","960x1088 (0.88)","960x1024 (0.94)","1024x1024 (1.0)","1024x960 (1.07)","1088x960 (1.13)","1088x896 (1.21)","1152x896 (1.29)","1152x832 (1.38)","1216x832 (1.46)","1280x768 (1.67)","1344x768 (1.75)","1344x704 (1.91)","1408x704 (2.0)","1472x704 (2.09)","1536x640 (2.4)","1600x640 (2.5)","1664x576 (2.89)","1728x576 (3.0)",], {"default": "1024x1024 (1.0)"}), + }} + + RETURN_TYPES = ("INT","INT",) + RETURN_NAMES = ("width", "height",) + FUNCTION = "execute" + CATEGORY = "essentials" + + def execute(self, resolution): + width, height = resolution.split(" ")[0].split("x") + + return (width, height,) + NODE_CLASS_MAPPINGS = { "StableZero123_Increments": StableZero123_Increments, @@ -913,6 +945,9 @@ NODE_CLASS_MAPPINGS = { "ModelCompile+": ModelCompile, "BatchCount+": BatchCount, + + "CLIPTextEncodeSDXL+": CLIPTextEncodeSDXLSimplified, + "SDXLResolutionPicker+": SDXLResolutionPicker, } NODE_DISPLAY_NAME_MAPPINGS = { @@ -929,7 +964,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ImageEnhanceDifference+": "🔧 Image Enhance Difference", "ImageExpandBatch+": "🔧 Image Expand Batch", "ImageFromBatch+": "🔧 Image From Batch", - "ExtractKeyframes+": "🔧 Extract Keyframes", + "ExtractKeyframes+": "🔧 Extract Keyframes (experimental)", "MaskBlur+": "🔧 Mask Blur", "MaskFlip+": "🔧 Mask Flip", @@ -945,4 +980,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "ModelCompile+": "🔧 Compile Model", "BatchCount+": "🔧 Batch Count", + + "CLIPTextEncodeSDXL+": "🔧 SDXLCLIPTextEncode", + "SDXLResolutionPicker+": "🔧 SDXL Resolutions", }