diff --git a/README.MD b/README.MD index bc67783..0142302 100644 --- a/README.MD +++ b/README.MD @@ -26,10 +26,8 @@ The intention for this repository is to build a suite of nodes that can be used - **QuickShapeMask**: Rapidly generate shape masks (circle, square) with customizable dimensions. - **DTypeConverter**: Convert masks between different data types (float16, uint8, float32, float64). - **FastWebcamCapture**: High-performance webcam capture node with resizing capabilities. - -### VAE Tools 🖼️ -- **TAESDVaeEncode**: TAESD VAE encoding node. -- **TAESDVaeDecode**: TAESD VAE decoding node. +- **SimilarityFilter**: Filter out similar consecutive images and control downstream execution. Perfect for optimizing real-time workflows by skipping redundant processing of similar frames. +- **LazyCondition**: Powerful conditional execution node that supports any input type. Uses lazy evaluation to truly skip execution of unused paths and maintains state to avoid feedback loops. ## Movement Patterns 🔄 @@ -61,6 +59,12 @@ Sequence controls allow you to specify exact values to cycle through. You can co ### FPS Monitor Outputs an image and mask showing current and average FPS. Useful for performance monitoring in real-time workflows. +### Utility Controls +Use utility nodes to optimize and control your workflow: +- **FPS Monitor**: Monitor performance with a visual overlay +- **SimilarityFilter**: Skip processing of similar frames by comparing consecutive images. Great for optimizing real-time workflows by only processing frames that have meaningful changes. +- **LazyCondition**: Create conditional execution paths that truly skip processing of unused branches. Works with any input type (images, latents, text, numbers) and maintains state of the last successful output to avoid feedback loops. + ## Examples 🎬 ### Value Control Demo diff --git a/__init__.py b/__init__.py index c977445..6db2b40 100644 --- a/__init__.py +++ b/__init__.py @@ -1,9 +1,8 @@ from .controls.value_controls import FloatControl, IntControl, StringControl from .controls.sequence_controls import FloatSequence, IntSequence, StringSequence -from .controls.utility_controls import FPSMonitor +from .controls.utility_controls import FPSMonitor, SimilarityFilter, LazyCondition from .controls.motion_controls import MotionController, ROINode, IntegerMotionController from .quick_shape_mask import QuickShapeMask -from .tiny_vae import TAESDVaeEncode, TAESDVaeDecode from .misc_nodes import DTypeConverter, FastWebcamCapture NODE_CLASS_MAPPINGS = { @@ -14,14 +13,18 @@ NODE_CLASS_MAPPINGS = { "IntSequence": IntSequence, "StringSequence": StringSequence, "FPSMonitor": FPSMonitor, + "SimilarityFilter": SimilarityFilter, + "LazyCondition": LazyCondition, "MotionController": MotionController, "IntegerMotionController": IntegerMotionController, + + + + "ROINode": ROINode, #"IntervalControl": IntervalCo ntrol, #"DeltaControl": DeltaControl, "QuickShapeMask": QuickShapeMask, - "TAESDVaeEncode": TAESDVaeEncode, - "TAESDVaeDecode": TAESDVaeDecode, "DTypeConverter": DTypeConverter, "FastWebcamCapture": FastWebcamCapture, } @@ -44,8 +47,13 @@ NODE_DISPLAY_NAME_MAPPINGS = { "TAESDVaeDecode": "TAESD VAE Decode (RyanOnTheInside)", "DTypeConverter": "DType Converter (RyanOnTheInside)", "FastWebcamCapture": "Fast Webcam Capture (RyanOnTheInside)", + "SimilarityFilter": "Similarity Filter (RyanOnTheInside)", + "LazyCondition": "Lazy Condition (RyanOnTheInside)", + + } + WEB_DIRECTORY = "./web/js" __all__ = ['NODE_CLASS_MAPPINGS', 'NODE_DISPLAY_NAME_MAPPINGS'] \ No newline at end of file diff --git a/controls/similar_image_filter.py b/controls/similar_image_filter.py new file mode 100644 index 0000000..66a4481 --- /dev/null +++ b/controls/similar_image_filter.py @@ -0,0 +1,44 @@ +import torch +from typing import Optional +import random + +class SimilarImageFilter: + def __init__(self, threshold: float = 0.98, max_skip_frame: float = 10) -> None: + self.threshold = threshold + + self.prev_tensor = None + self.cos = torch.nn.CosineSimilarity(dim=0, eps=1e-6) + self.max_skip_frame = max_skip_frame + self.skip_count = 0 + + def __call__(self, x: torch.Tensor) -> Optional[torch.Tensor]: + if self.prev_tensor is None: + self.prev_tensor = x.detach().clone() + return x + else: + cos_sim = self.cos(self.prev_tensor.reshape(-1), x.reshape(-1)).item() + sample = random.uniform(0, 1) + if self.threshold >= 1: + skip_prob = 0 + else: + skip_prob = max(0, 1 - (1 - cos_sim) / (1 - self.threshold)) + + # not skip frame + if skip_prob < sample: + self.prev_tensor = x.detach().clone() + return x + # skip frame + else: + if self.skip_count > self.max_skip_frame: + self.skip_count = 0 + self.prev_tensor = x.detach().clone() + return x + else: + self.skip_count += 1 + return None + + def set_threshold(self, threshold: float) -> None: + self.threshold = threshold + + def set_max_skip_frame(self, max_skip_frame: float) -> None: + self.max_skip_frame = max_skip_frame \ No newline at end of file diff --git a/controls/utility_controls.py b/controls/utility_controls.py index 927d07d..92562c2 100644 --- a/controls/utility_controls.py +++ b/controls/utility_controls.py @@ -1,8 +1,15 @@ -from ..base.control_base import ControlNodeBase +import os +import sys import time import numpy as np import cv2 import torch +import random + +# Add package root to Python path +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from base.control_base import ControlNodeBase +from controls.similar_image_filter import SimilarImageFilter class FPSMonitor(ControlNodeBase): """Generates an FPS overlay as an image and mask""" @@ -121,3 +128,154 @@ class FPSMonitor(ControlNodeBase): self.set_state(state) return (state["cached_image"], state["cached_mask"]) + +class SimilarityFilter(ControlNodeBase): + DESCRIPTION = "Filters out similar consecutive images and outputs a signal to control downstream execution." + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image": ("IMAGE", { + "tooltip": "Input image to compare with previous frame" + }), + "always_execute": ("BOOLEAN", { + "default": False, + }), + "threshold": ("FLOAT", { + "default": 0.98, + "min": 0.0, + "max": 1.0, + "step": 0.01, + "tooltip": "Similarity threshold (0-1). Higher values mean more frames are considered similar" + }), + "max_skip_frames": ("INT", { + "default": 10, + "min": 1, + "max": 100, + "step": 1, + "tooltip": "Maximum number of consecutive frames to skip before forcing execution" + }), + } + } + + RETURN_TYPES = ("IMAGE", "BOOLEAN") + RETURN_NAMES = ("image", "should_execute") + FUNCTION = "update" + CATEGORY = "real-time/control/utility" + + def __init__(self): + super().__init__() + self._similarity_filter = SimilarImageFilter() + + def update(self, image, threshold=0.98, max_skip_frames=10, always_execute=False): + print(f"[DEBUG] Input image object: {hex(id(image))}, shape: {image.shape}, device: {image.device}") + + # Get state with defaults + state = self.get_state({ + "prev_image": None, + "skip_count": 0 + }) + + # First frame case + if state["prev_image"] is None: + state["prev_image"] = image.detach().clone() + state["skip_count"] = 0 + self.set_state(state) + return (image, True) # Always execute first frame + + # Update filter parameters + self._similarity_filter.set_threshold(threshold) + self._similarity_filter.set_max_skip_frame(max_skip_frames) + + # Use filter to check similarity + result = self._similarity_filter(image) + should_execute = result is not None + + # If we should skip (probability check) + if not should_execute: + # Check if we've skipped too many frames + if state["skip_count"] >= max_skip_frames: + state["prev_image"] = image.detach().clone() + state["skip_count"] = 0 + self.set_state(state) + return (image, True) # Force execution after max skips + + # Skip frame - return previous image and False for execution + state["skip_count"] += 1 + self.set_state(state) + return (state["prev_image"], False) + + # Frame is different enough - process it + state["prev_image"] = image.detach().clone() + state["skip_count"] = 0 + self.set_state(state) + return (image, True) + +class AlwaysEqualProxy(str): + #borrowed from https://github.com/theUpsider/ComfyUI-Logic + def __eq__(self, _): + return True + + def __ne__(self, _): + return False + +class LazyCondition(ControlNodeBase): + DESCRIPTION = "Uses lazy evaluation to truly skip execution of unused paths. Maintains state of the last value to circumvent feedback loops." + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "condition": (AlwaysEqualProxy("*"), { + "tooltip": "When truthy (non-zero, non-empty, non-None), evaluates and returns if_true path. When falsy, returns either fallback or previous state of if_true.", + "forceInput": True, + }), + "if_true": (AlwaysEqualProxy("*"), { + "lazy": True, + "tooltip": "The path that should only be evaluated when condition is truthy" + }), + "fallback": (AlwaysEqualProxy("*"), { + "tooltip": "Alternative value to use when condition is falsy or no previous state of if_true" + }), + "use_fallback": ("BOOLEAN", { + "default": False, + "tooltip": "When False, uses last successful if_true result (if one exists). When True, uses fallback value" + }), + } + } + + RETURN_TYPES = (AlwaysEqualProxy("*"),) + FUNCTION = "update" + CATEGORY = "real-time/control/utility" + + def check_lazy_status(self, condition, if_true, fallback, use_fallback): + """Only evaluate the if_true path if condition is truthy.""" + needed = ["fallback"] # Always need the fallback value + if condition: + needed.append("if_true") + return needed + + def update(self, condition, if_true, fallback, use_fallback): + """Route to either if_true output or fallback value, maintaining state of last if_true.""" + state = self.get_state({ + "prev_output": None + }) + + if condition: # Let Python handle truthiness + # Update last state when we run if_true path + state["prev_output"] = if_true if if_true is not None else fallback + if hasattr(if_true, 'detach'): + state["prev_output"] = if_true.detach().clone() + self.set_state(state) + return (if_true,) + else: + if use_fallback or state["prev_output"] is None: + return (fallback,) + else: + return (state["prev_output"],) + + + + + diff --git a/examples/on_off_example_api.json b/examples/on_off_example_api.json new file mode 100644 index 0000000..55de548 --- /dev/null +++ b/examples/on_off_example_api.json @@ -0,0 +1,178 @@ +{ + "568": { + "inputs": { + "always_execute": true, + "threshold": 0.1, + "blur_size": 5, + "minimum_value": 0, + "maximum_value": 1, + "starting_value": 0, + "rounding_mode": "round", + "image": [ + "583", + 0 + ], + "roi_chain": [ + "569", + 0 + ] + }, + "class_type": "IntegerMotionController", + "_meta": { + "title": "Integer Motion Controller (RyanOnTheInside)" + } + }, + "569": { + "inputs": { + "action": "toggle", + "value": 1, + "mask": [ + "584", + 0 + ] + }, + "class_type": "ROINode", + "_meta": { + "title": "ROI Node (RyanOnTheInside)" + } + }, + "572": { + "inputs": { + "value": 0.4, + "width": 512, + "height": 512 + }, + "class_type": "SolidMask", + "_meta": { + "title": "SolidMask" + } + }, + "573": { + "inputs": { + "x": 0, + "y": 0, + "operation": "multiply", + "destination": [ + "584", + 0 + ], + "source": [ + "572", + 0 + ] + }, + "class_type": "MaskComposite", + "_meta": { + "title": "MaskComposite" + } + }, + "574": { + "inputs": { + "mask": [ + "573", + 0 + ] + }, + "class_type": "MaskToImage", + "_meta": { + "title": "Convert Mask to Image" + } + }, + "575": { + "inputs": { + "images": [ + "579", + 0 + ] + }, + "class_type": "PreviewImage", + "_meta": { + "title": "Preview Image" + } + }, + "576": { + "inputs": { + "kernel_size": 61, + "sigma": 5, + "mode": "cuda", + "images": [ + "583", + 0 + ] + }, + "class_type": "GaussianBlur", + "_meta": { + "title": "GaussianBlur" + } + }, + "579": { + "inputs": { + "x": 0, + "y": 0, + "resize_source": false, + "destination": [ + "587", + 0 + ], + "source": [ + "574", + 0 + ], + "mask": [ + "573", + 0 + ] + }, + "class_type": "ImageCompositeMasked", + "_meta": { + "title": "ImageCompositeMasked" + } + }, + "583": { + "inputs": { + "image": "241228022022_scene1_00033_.png", + "upload": "image" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "584": { + "inputs": { + "shape": "circle", + "width": 64, + "height": 64, + "x": 437, + "y": 75, + "canvas_width": 512, + "canvas_height": 512, + "batch_size": 1 + }, + "class_type": "QuickShapeMask", + "_meta": { + "title": "Quick Shape Mask (RyanOnTheInside)" + } + }, + "587": { + "inputs": { + "use_fallback": true, + "condition": [ + "568", + 0 + ], + "if_true": [ + "576", + 0 + ], + "fallback": [ + "583", + 0 + ] + }, + "class_type": "LazyCondition", + "_meta": { + "title": "Lazy Condition (RyanOnTheInside)" + } + } +} \ No newline at end of file diff --git a/examples/similarity_filter_example_api.json b/examples/similarity_filter_example_api.json new file mode 100644 index 0000000..41e9a5b --- /dev/null +++ b/examples/similarity_filter_example_api.json @@ -0,0 +1,75 @@ +{ + "575": { + "inputs": { + "images": [ + "587", + 0 + ] + }, + "class_type": "PreviewImage", + "_meta": { + "title": "Preview Image" + } + }, + "576": { + "inputs": { + "kernel_size": 61, + "sigma": 5, + "mode": "cuda", + "images": [ + "588", + 0 + ] + }, + "class_type": "GaussianBlur", + "_meta": { + "title": "GaussianBlur" + } + }, + "583": { + "inputs": { + "image": "241228022022_scene1_00033_.png", + "upload": "image" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "587": { + "inputs": { + "use_fallback": false, + "condition": [ + "588", + 1 + ], + "if_true": [ + "576", + 0 + ], + "fallback": [ + "588", + 0 + ] + }, + "class_type": "LazyCondition", + "_meta": { + "title": "Lazy Condition (RyanOnTheInside)" + } + }, + "588": { + "inputs": { + "always_execute": false, + "threshold": 0.98, + "max_skip_frames": 10, + "image": [ + "583", + 0 + ] + }, + "class_type": "SimilarityFilter", + "_meta": { + "title": "Similarity Filter (RyanOnTheInside)" + } + } +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6b7ea0e..dfb6d44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui_realtimenodes" description = "These nodes are for real-time applications of ComfyUI." -version = "1.0.0" +version = "1.1.0" license = {file = "LICENSE"} dependencies = ["opencv-python"] diff --git a/tiny_vae.py b/tiny_vae.py deleted file mode 100644 index 78ec45b..0000000 --- a/tiny_vae.py +++ /dev/null @@ -1,86 +0,0 @@ -from typing import Tuple - -import PIL -import numpy as np -import torch -from PIL import Image -from torch import Tensor - -import folder_paths -from comfy import model_management -from comfy.taesd.taesd import TAESD - -#credit for Decode goes to https://github.com/M1kep/ComfyUI-OtherVAEs, this is based on that implementation, but fixes some issues and includes the encoder -class TAESDVaeDecode: - @classmethod - def INPUT_TYPES(cls): # type: ignore - return { - "required": { - "latent": ("LATENT",), - "vae": (folder_paths.get_filename_list("vae_approx"), {}) - } - } - - RETURN_TYPES = ("IMAGE",) - FUNCTION = "decode" - OUTPUT_IS_LIST = (False,) - CATEGORY = "conditioning" - - def __init__(self): - self.taesd = None - - def decode(self, latent: torch.Tensor, vae: str) -> Tuple[torch.Tensor]: - if self.taesd is None: - self.taesd = TAESD(None, folder_paths.get_full_path("vae_approx", vae)).to(model_management.get_torch_device()) - - x_sample = self.taesd.taesd_decoder((latent['samples'].to(model_management.get_torch_device()) * 0.18215))[0].detach() - x_sample = x_sample.sub(0.5).mul(2) - x_sample = torch.clamp((x_sample + 1.0) / 2.0, min=0.0, max=1.0) - # x_sample = x_sample.movedim(1, -1) - x_sample = x_sample.permute(1, 2, 0) - - # x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2) - # x_sample = x_sample.astype(np.uint8) - - return (torch.unsqueeze(x_sample, 0),) - -class TAESDVaeEncode: - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "image": ("IMAGE",), # ComfyUI image input (NHWC format) - "vae": (folder_paths.get_filename_list("vae_approx"), {}) - } - } - - RETURN_TYPES = ("LATENT",) - FUNCTION = "encode" - OUTPUT_IS_LIST = (False,) - CATEGORY = "conditioning" - - def __init__(self): - self.taesd = None - - def encode(self, image: torch.Tensor, vae: str) -> Tuple[dict]: - if self.taesd is None: - # Initialize with encoder path only since we only need encoding - self.taesd = TAESD( - encoder_path=folder_paths.get_full_path("vae_approx", vae), - decoder_path=None - ).to(model_management.get_torch_device()) - - # Move input to the same device as the model - device = model_management.get_torch_device() - x_sample = image.to(device) - - # Convert NHWC to NCHW format for the encoder - x_sample = x_sample.permute(0, 3, 1, 2) - - # Scale input from [0,1] to [-1,1] - x_sample = (x_sample * 2.0) - 1.0 - - # Encode the image using TAESD's encode method - latent = self.taesd.encode(x_sample) - - return ({"samples": latent},) diff --git a/web/js/widget_validation.js b/web/js/widget_validation.js index c686d8c..7ddaf9e 100644 --- a/web/js/widget_validation.js +++ b/web/js/widget_validation.js @@ -3,6 +3,8 @@ import { app } from "../../../scripts/app.js"; // TODO: Add validation for step values to ensure they match target widget constraints +// List of widget names that need validation +const VALIDATED_WIDGETS = ["values", "maximum_value", "minimum_value", "starting_value"]; // Register validation behavior when nodes are connected app.registerExtension({ @@ -12,6 +14,18 @@ app.registerExtension({ return; } + // Check if node has any widgets that need validation + const hasValidatedWidgets = nodeData.input?.required && + Object.entries(nodeData.input.required).some(([name, config]) => + VALIDATED_WIDGETS.includes(name) || + (typeof config === "object" && config[0] === "FLOAT") || + (typeof config === "object" && config[0] === "INT") + ); + + if (!hasValidatedWidgets) { + return; + } + // Store constraints from connected widgets nodeType.prototype.targetConstraints = null;