diff --git a/README.md b/README.md index d42c1df..07f6c28 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ The core functionality is taken from [comfyui_lora_tag_loader](https://github.co ### Model Unloader nodes -A node that automatically unloads all checkpoints from memory. It must be added to a sequential chain of nodes in the workflow. There are three versions of this node: Hard (complete unloading of all checkpoints from memory, except for GGUF (not supported yet)), Middle (the same as Hard, but in the future I plan to add widgets with the ability to select a mode), Soft (without unloading checkpoints from memory, just soft cleaning of memory from garbage). +A node that automatically unloads all models from memory. It must be added to a sequential chain of nodes in the workflow. There are three versions of this node: Hard (complete unloading of all models from memory), Soft (without unloading models from memory, just soft cleaning of memory from garbage), Model Unload (unloading of specifiy model from memory), and "Soft Full Clean RAM and VRAM" for the case when it is necessary to unload all models from memory and clean memory from garbage after the workflow is completed (the task in the queue is finished). ### File Counter diff --git a/node_list.json b/node_list.json index 6a64e3e..16d37bb 100644 --- a/node_list.json +++ b/node_list.json @@ -47,11 +47,13 @@ "Text To List [LP]":"Convert many texts to list of texts", "Split Compound Text [LP]":"Split Compound Text", "Keep Only English Words [LP]":"Removes all words with symbols that do not correspond to English symbols. The entire word is removed (from space to space), regardless of whether it contains English symbols along with other symbols. The node is necessary for excluding any words and symbols from the text except English.", - "Model Unloader [LP]":"Unloads all neural models that are in the model manager from VRAM. Does not unload models that are loaded into VRAM bypassing the model manager.", - "Soft Model Unloader [LP]":"Unloads from VRAM those files and cache that are not currently in use. Does not unload models.", - "Hard Model Unloader [LP]":"Unloads all neural models that are in the model manager from VRAM. Does not unload models that are loaded into VRAM bypassing the model manager.", - "Override CLIP Device [LP]":"Changes the main computing processor and the RAM used for CLIP. You can specify any video card or processor that is available in the system.", - "Override VAE Device [LP]":"Changes the main computing processor and the RAM used for VAE. You can specify any video card or processor that is available in the system.", + "Unload Model [LP]":"Unload a specific specified model from VRAM. The node can be used anywhere in the workflow where you need to unload a specific model.", + "Soft Unload Models Data [LP]":"Unloads from VRAM those files and cache that are not currently in use. Does not unload models.", + "Hard Unload All Models [LP]":"Force unloads all neural models that are in the model manager from VRAM. Does not unload models that are loaded into VRAM bypassing the model manager of ComfyUI.", + "Soft Full Clean RAM and VRAM [LP]":"Completely clears all models and cached data from RAM and VRAM upon completion of the current workflow in the assigned queue task. Can be placed anywhere in the workflow, but will only work after the current workflow in the current queue task has completed. This node is useful when workflows and their nodes leave a lot of garbage in RAM and VRAM during operation, disrupting the operation of subsequent workflows in the queue tasks.", + "Override CLIP Device [LP]":"Changes the main computing processor and the RAM used for CLIP. You can specify any video card or processor that is available in the system. ATTENTION! When using this node, manual cleaning of models from memory using nodes that clean models from VRAM/RAM memory stops working!", + "Override VAE Device [LP]":"Changes the main computing processor and the RAM used for VAE. You can specify any video card or processor that is available in the system. ATTENTION! When using this node, manual cleaning of models from memory using nodes that clean models from VRAM/RAM memory stops working!", + "Override CLIP Vision Device [LP]":"Changes the main computing processor and the RAM used for CLIP Vision. You can specify any video card or processor that is available in the system. ATTENTION! When using this node, manual cleaning of models from memory using nodes that clean models from VRAM/RAM memory stops working!", "Delay [LP]":"Delay", "Seed [LP]":"Seed", "String Cycler [LP]":"String Cycler", diff --git a/nodes/unloaders/model_unloaders_LP.py b/nodes/unloaders/model_unloaders_LP.py index 05e52b9..f1672af 100644 --- a/nodes/unloaders/model_unloaders_LP.py +++ b/nodes/unloaders/model_unloaders_LP.py @@ -1,6 +1,8 @@ from comfy import model_management import gc import torch +import requests +from comfy.cli_args import args as comfy_args class AnyType(str): def __ne__(self, __value: object) -> bool: @@ -13,34 +15,42 @@ class ModelUnloader: pass @classmethod - def INPUT_TYPES(s): - return { - "required": { - "source": (any, {}), - }, + def INPUT_TYPES(cls): + return { + "required": {"source": (any, )}, + "optional": {"model for unload": (any, )}, } - - RETURN_TYPES = (any,) - + + @classmethod + def VALIDATE_INPUTS(s, **kwargs): + return True + + RETURN_TYPES = (any, ) FUNCTION = "unload_model" - - OUTPUT_NODE = True - CATEGORY = "LevelPixel/Unloaders" - + OUTPUT_NODE = True + def unload_model(self, **kwargs): - loadedmodels=model_management.current_loaded_models - unloaded_model = False - for i in range(len(loadedmodels) -1, -1, -1): - m = loadedmodels.pop(i) - m.model_unload() - del m - unloaded_model = True - if unloaded_model: - model_management.soft_empty_cache() - gc.collect() - torch.cuda.empty_cache() - return (kwargs["source"],) + loaded_models = model_management.loaded_models() + if kwargs.get("source") in loaded_models: + loaded_models.remove(kwargs.get("model for unload")) + else: + model = kwargs.get("model for unload") + if type(model) == dict: + keys = [(key, type(value).__name__) for key, value in model.items()] + for key, model_type in keys: + if key == 'model': + print(f"Unloading model of type {model_type}") + del model[key] + model_management.free_memory(1e30, model_management.get_torch_device(), loaded_models) + model_management.soft_empty_cache(True) + try: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except: + print("Unable to clear cache") + return (kwargs.get("source"),) class SoftModelUnloader: def __init__(self): @@ -55,11 +65,8 @@ class SoftModelUnloader: } RETURN_TYPES = (any,) - FUNCTION = "soft_unload_model" - OUTPUT_NODE = True - CATEGORY = "LevelPixel/Unloaders" def soft_unload_model(self, **kwargs): @@ -73,44 +80,103 @@ class HardModelUnloader: pass @classmethod - def INPUT_TYPES(s): - return { - "required": { - "source": (any, {}), - }, + def INPUT_TYPES(cls): + return { + "required": {"source": (any, )}, } - - RETURN_TYPES = (any,) - + + @classmethod + def VALIDATE_INPUTS(s, **kwargs): + return True + + RETURN_TYPES = (any, ) FUNCTION = "hard_unload_model" - - OUTPUT_NODE = True - CATEGORY = "LevelPixel/Unloaders" - + OUTPUT_NODE = True + def hard_unload_model(self, **kwargs): - loadedmodels=model_management.current_loaded_models - unloaded_model = False + print("Unload Models") + loadedmodels = model_management.current_loaded_models for i in range(len(loadedmodels) -1, -1, -1): m = loadedmodels.pop(i) m.model_unload() del m - unloaded_model = True - if unloaded_model: - model_management.soft_empty_cache() - gc.collect() - torch.cuda.empty_cache() + + try: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except: + print("Unable to clear cache") + + model_management.unload_all_models() + model_management.soft_empty_cache(True) + + try: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except: + print("Unable to clear cache") + return (kwargs.get("source"),) + +def _server_base_url(): + tls = bool(getattr(comfy_args, "tls_keyfile", None) and getattr(comfy_args, "tls_certfile", None)) + scheme = "https" if tls else "http" + + port = getattr(comfy_args, "port", 8188) + listen = str(getattr(comfy_args, "listen", "127.0.0.1") or "127.0.0.1") + + hosts = [h.strip() for h in listen.split(",") if h.strip()] + def is_wildcard(h): return h in ("0.0.0.0", "::") + host = next((h for h in hosts if not is_wildcard(h)), None) or "127.0.0.1" + + if ":" in host and not host.startswith("["): + host = f"[{host}]" + + return f"{scheme}://{host}:{port}" + +class SoftFullCleanRAMAndVRAM: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "source": (any, {}), + "free_execution_cache": ("BOOLEAN", {"default": True}), + } + } + + RETURN_TYPES = (any,) + FUNCTION = "soft_free_models" + CATEGORY = "LevelPixel/Unloaders" + OUTPUT_NODE = True + + def soft_free_models(self, free_execution_cache, **kwargs): + + url = f"{_server_base_url()}/free" + if free_execution_cache: + payload = {"unload_models": True, "free_memory": True} + else: + payload = {"unload_models": True} + + res = requests.post(url, json=payload) + if res.status_code == 200: + print("Models unloaded (and execution cache cleared)" if free_execution_cache else " Models unloaded") + else: + print("Failed to unload models. Maybe outdated ComfyUI version.") return (kwargs["source"],) NODE_CLASS_MAPPINGS = { "ModelUnloader|LP": ModelUnloader, "SoftModelUnloader|LP": SoftModelUnloader, "HardModelUnloader|LP": HardModelUnloader, + "SoftFullCleanRAMAndVRAM|LP": SoftFullCleanRAMAndVRAM, } NODE_DISPLAY_NAME_MAPPINGS = { - "ModelUnloader|LP": "Model Unloader [LP]", - "SoftModelUnloader|LP": "Soft Model Unloader [LP]", - "HardModelUnloader|LP": "Hard Model Unloader [LP]", + "ModelUnloader|LP": "Unload Model [LP]", + "SoftModelUnloader|LP": "Soft Unload Models Data [LP]", + "HardModelUnloader|LP": "Hard Unload All Models [LP]", + "SoftFullCleanRAMAndVRAM|LP": "Soft Full Clean RAM and VRAM [LP]", } diff --git a/nodes/unloaders/override_device_LP.py b/nodes/unloaders/override_device_LP.py index af827b0..c35f95e 100644 --- a/nodes/unloaders/override_device_LP.py +++ b/nodes/unloaders/override_device_LP.py @@ -1,11 +1,10 @@ import types import torch -import comfy.model_management class OverrideDevice: @classmethod def INPUT_TYPES(s): - devices = ["cpu",] + devices = ["auto","cpu",] for k in range(0, torch.cuda.device_count()): devices.append(f"cuda:{k}") @@ -19,8 +18,13 @@ class OverrideDevice: CATEGORY = "LevelPixel/Unloaders" def override(self, model, model_attr, device): + if device == "auto": + return (model,) + + torch.device(device) + model.device = device - patcher = getattr(model, "patcher", model) #.clone() + patcher = getattr(model, "patcher", model) for name in ["device", "load_device", "offload_device", "current_device", "output_device"]: setattr(patcher, name, device) @@ -47,7 +51,7 @@ class OverrideCLIPDevice(OverrideDevice): CATEGORY = "LevelPixel/Unloaders" def patch(self, clip, device): - return self.override(clip, "cond_stage_model", torch.device(device)) + return self.override(clip, "cond_stage_model", device) class OverrideVAEDevice(OverrideDevice): @classmethod @@ -60,15 +64,29 @@ class OverrideVAEDevice(OverrideDevice): CATEGORY = "LevelPixel/Unloaders" def patch(self, vae, device): - return self.override(vae, "first_stage_model", torch.device(device)) + return self.override(vae, "first_stage_model", device) +class OverrideCLIPVisionDevice(OverrideDevice): + @classmethod + def INPUT_TYPES(s): + k = super().INPUT_TYPES() + k["required"]["clip_vision"] = ("CLIP_VISION",) + return k + + RETURN_TYPES = ("CLIP_VISION",) + CATEGORY = "LevelPixel/Unloaders" + + def patch(self, clip_vision, device): + return self.override(clip_vision, "model", device) NODE_CLASS_MAPPINGS = { "OverrideCLIPDevice|LP": OverrideCLIPDevice, "OverrideVAEDevice|LP": OverrideVAEDevice, + "OverrideCLIPVisionDevice|LP": OverrideCLIPVisionDevice, } NODE_DISPLAY_NAME_MAPPINGS = { "OverrideCLIPDevice|LP": "Override CLIP Device [LP]", "OverrideVAEDevice|LP": "Override VAE Device [LP]", -} + "OverrideCLIPVisionDevice|LP": "Override CLIP Vision Device [LP]", +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index effadd7..6a73b98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-levelpixel" -description = "Main nodes of the Level Pixel company (levelpixel, LP). In this package you will find easy-to-use atomic nodes that can help automate various processes - memory cleaning automation (RAM free, VRAM free), Lora loading like in Automatic1111, powerful inpaint crop and stitch, pipeline, declaring standard variables, converting variables, google translate, filtering tags by categories, working with file system, counting files in file system, counting objects, counting images. The names of the key nodes in this package are: Inpaint-CropAndStitch, Inpaint Crop, Inpaint Stitch, Resize Image, Resize Mask, Model Unloader, Free memory, Tag Filters, Tag Category Filters, Tag Choice Parser, File counter, Image Loader From Path (with counters), Google Translator, Language Translator, Override Device, Delay, Lora Tag Loader, Pipeline, Resize Masks, Resize Image, Iterators. Licensed under GNU GPLv3. Repository: https://github.com/LevelPixel/ComfyUI-LevelPixel" -version = "1.3.2" +description = "Main nodes of the Level Pixel company (levelpixel, LP). In this package you will find easy-to-use atomic nodes that can help automate various processes - memory cleaning automation (RAM free, VRAM free), Model unload, Lora loading like in Automatic1111, powerful inpaint crop and stitch, pipeline, declaring standard variables, converting variables, google translate, filtering tags by categories, working with file system, counting files in file system, counting objects, counting images. The names of the key nodes in this package are: Inpaint-CropAndStitch, Inpaint Crop, Inpaint Stitch, Resize Image, Resize Mask, Model Unloader, Free memory, Tag Filters, Tag Category Filters, Tag Choice Parser, File counter, Image Loader From Path (with counters), Google Translator, Language Translator, Override Device, Delay, Lora Tag Loader, Pipeline, Resize Masks, Resize Image, Iterators. Licensed under GNU GPLv3. Repository: https://github.com/LevelPixel/ComfyUI-LevelPixel" +version = "1.3.3" license = { file = "LICENSE.txt" } dependencies = [ "torch>=2.0.1",