add:easy LLLiteLoader node

This commit is contained in:
yolain
2023-12-13 21:39:47 +08:00
parent c8521c96ab
commit 22ce1f9f36
10 changed files with 419 additions and 38 deletions
+6
View File
@@ -14,12 +14,18 @@
"easy controlnetLoader": {
"title": "简易Controlnet"
},
"easy LLLite": {
"title": "简易LLLite"
},
"easy globalSeed": {
"title": "全局Seed"
},
"easy preSampling": {
"title": "预采样参数(基础)"
},
"easy preSamplingAdvanced": {
"title": "预采样参数(高级)"
},
"easy preSamplingSdTurbo": {
"title": "预采样参数(SdTurbo)"
},
+10 -1
View File
@@ -19,7 +19,15 @@ EasyUse is simplified on the basis of [tinyterraNodes](https://github.com/TinyTe
### Updated
- **[Updated 12/11/2023]** Added the `showSpentTime` node to display the time spent on image diffusion and the time spent on VAE decoding images
**[Updated at 12/13/2023]**
- Added the `easy LLLiteLoader` node, if you have pre-installed the kohya-ss/ControlNet-LLLite-ComfyUI package, please move the model files in the models to `ComfyUI\models\controlnet\` (i.e. in the default controlnet path of comfy, please do not change the file name of the model, otherwise it will not be read).
- Modify `easy controlnetLoader` to the bottom of the loader category.
- Added size display for `easy imageSize` and `easy imageSizeByLongerSize` outputs.
**[Updated at 12/11/2023]**
- Added the `showSpentTime` node to display the time spent on image diffusion and the time spent on VAE decoding images
### Major optimizations
@@ -34,6 +42,7 @@ Disclaimer: Opened source was not easy. I have a lot of respect for the contribu
|:---------------------------|:----------------------------------------------------------------------------|:----------------------------------|
| easy SetNode | [diffus3/ComfyUI-extensions](https://github.com/diffus3/ComfyUI-extensions) | diffus3.SetNode |
| easy GetNode | [diffus3/ComfyUI-extensions](https://github.com/diffus3/ComfyUI-extensions) | diffus3.GetNode |
| easy LLLiteLoader | [kohya-ss/ControlNet-LLLite-ComfyUI](https://github.com/kohya-ss/ControlNet-LLLite-ComfyUI) | LLLiteLoader |
| easy GlobalSeed | [ltdrdata/ComfyUI-Inspire-Pack](https://github.com/ltdrdata/ComfyUI-Inspire-Pack) | Global Seed (Inspire) |
| easy PreSamplingDynamicCFG | [sd-dynamic-thresholding](https://github.com/mcmonkeyprojects/sd-dynamic-thresholding) | DynamicThresholdingFull |
| DynamicThresholdingFull | [sd-dynamic-thresholding](https://github.com/mcmonkeyprojects/sd-dynamic-thresholding) | DynamicThresholdingFull |
+17 -8
View File
@@ -8,7 +8,7 @@
为了更加方便简单地使用ComfyUI,我对一部分常用的节点做了一些优化与整合。
[![Bilibili Badge](https://img.shields.io/badge/使用说明视频-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white&link=https://www.bilibili.com/video/BV1vQ4y1G7z7)](https://www.bilibili.com/video/BV1vQ4y1G7z7/)
[![Bilibili Badge](https://img.shields.io/badge/视频介绍-00A1D6?style=for-the-badge&logo=bilibili&logoColor=white&link=https://www.bilibili.com/video/BV1vQ4y1G7z7)](https://www.bilibili.com/video/BV1vQ4y1G7z7/)
</div>
## 流程对比
@@ -19,7 +19,15 @@ EasyUse 在 [tinyterraNodes](https://github.com/TinyTerra/ComfyUI_tinyterraNodes
### 更新
- **[2023-12-11]** 新增 `showSpentTime` 节点用于展示图片推理花费时间与VAE解码花费时间。
**2023-12-13**
- 新增 `easy LLLiteLoader` 节点,如果您预先安装过 kohya-ss/ControlNet-LLLite-ComfyUI 包,请将 models 里的模型文件移动至 ComfyUI\models\controlnet\ (即comfy默认的controlnet路径里,请勿修改模型的文件名,不然会读取不到)。
- 修改 `easy controlnetLoader` 到 loader 分类底下。
- 新增 `easy imageSize` 和 `easy imageSizeByLongerSize` 输出的尺寸显示。
**2023-12-11**
- 新增 `easy showSpentTime` 节点用于展示图片推理花费时间与VAE解码花费时间。
### 主要的优化
@@ -30,14 +38,15 @@ EasyUse 在 [tinyterraNodes](https://github.com/TinyTerra/ComfyUI_tinyterraNodes
声明: 非常尊重这些原作者们的付出,开源不易,我仅仅只是做了一些整合与优化。
| 节点名 | 相关的库 | 库相关的节点 |
|:---------------------------|:----------------------------------------------------------------------------|:----------------------|
| easy SetNode | [diffus3/ComfyUI-extensions](https://github.com/diffus3/ComfyUI-extensions) | diffus3.SetNode |
| easy GetNode | [diffus3/ComfyUI-extensions](https://github.com/diffus3/ComfyUI-extensions) | diffus3.GetNode |
| easy GlobalSeed | [ltdrdata/ComfyUI-Inspire-Pack](https://github.com/ltdrdata/ComfyUI-Inspire-Pack) | Global Seed (Inspire) |
| 节点名 | 相关的库 | 库相关的节点 |
|:---------------------------|:----------------------------------------------------------------------------|:------------------------|
| easy SetNode | [diffus3/ComfyUI-extensions](https://github.com/diffus3/ComfyUI-extensions) | diffus3.SetNode |
| easy GetNode | [diffus3/ComfyUI-extensions](https://github.com/diffus3/ComfyUI-extensions) | diffus3.GetNode |
| easy LLLiteLoader | [kohya-ss/ControlNet-LLLite-ComfyUI](https://github.com/kohya-ss/ControlNet-LLLite-ComfyUI) | LLLiteLoader |
| easy GlobalSeed | [ltdrdata/ComfyUI-Inspire-Pack](https://github.com/ltdrdata/ComfyUI-Inspire-Pack) | Global Seed (Inspire) |
| easy PreSamplingDynamicCFG | [sd-dynamic-thresholding](https://github.com/mcmonkeyprojects/sd-dynamic-thresholding) | DynamicThresholdingFull |
| DynamicThresholdingFull | [sd-dynamic-thresholding](https://github.com/mcmonkeyprojects/sd-dynamic-thresholding) | DynamicThresholdingFull |
| easy ImageInsetCrop | [rgthree/rgthree-comfy](https://github.com/rgthree/rgthree-comfy) | ImageInsetCrop |
| easy ImageInsetCrop | [rgthree/rgthree-comfy](https://github.com/rgthree/rgthree-comfy) | ImageInsetCrop |
## 示例
+1
View File
@@ -8,6 +8,7 @@ node_list = [
"server",
"easyNodes",
"image",
"lllite"
]
NODE_CLASS_MAPPINGS = {}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.2 MiB

After

Width:  |  Height:  |  Size: 1.1 MiB

+1 -1
View File
@@ -973,7 +973,7 @@ class controlnetSimple:
OUTPUT_NODE = True
FUNCTION = "controlnetApply"
CATEGORY = "EasyUse/PreSampling"
CATEGORY = "EasyUse/Loader"
def controlnetApply(self, pipe, control_net_name, image, positive=None, negative=None, strength=1):
controlnet_path = folder_paths.get_full_path("controlnet", control_net_name)
+9 -5
View File
@@ -114,8 +114,10 @@ class imageSize:
def image_width_height(self, image):
image = tensor2pil(image)
if image.size:
return (image.size[0], image.size[1])
return (0, 0)
result = (image.size[0], image.size[1])
else:
result = (0, 0)
return {"ui": {"text": "Width: "+str(result[0])+" , Height: "+str(result[1])}, "result": result}
# 图像尺寸
class imageSizeByLongerSide:
@@ -140,10 +142,12 @@ class imageSizeByLongerSide:
image = tensor2pil(image)
if image.size:
if image.size[0] > image.size[1]:
return (image.size[0],)
result = (image.size[0],)
else:
return (image.size[1],)
return (0,)
result = (image.size[1],)
else:
result = (0,)
return {"ui": {"text": str(result[0])}, "result": result}
NODE_CLASS_MAPPINGS = {
"easy imageInsetCrop": imageInsetCrop,
+286
View File
@@ -0,0 +1,286 @@
import math
import torch
import os
import folder_paths
import comfy
def get_file_list(path):
return [file for file in os.listdir(path) if file != "put_models_here.txt" and "lllite" in file]
def extra_options_to_module_prefix(extra_options):
# extra_options = {'transformer_index': 2, 'block_index': 8, 'original_shape': [2, 4, 128, 128], 'block': ('input', 7), 'n_heads': 20, 'dim_head': 64}
# block is: [('input', 4), ('input', 5), ('input', 7), ('input', 8), ('middle', 0),
# ('output', 0), ('output', 1), ('output', 2), ('output', 3), ('output', 4), ('output', 5)]
# transformer_index is: [0, 1, 2, 3, 4, 5, 6, 7, 8], for each block
# block_index is: 0-1 or 0-9, depends on the block
# input 7 and 8, middle has 10 blocks
# make module name from extra_options
block = extra_options["block"]
block_index = extra_options["block_index"]
if block[0] == "input":
module_pfx = f"lllite_unet_input_blocks_{block[1]}_1_transformer_blocks_{block_index}"
elif block[0] == "middle":
module_pfx = f"lllite_unet_middle_block_1_transformer_blocks_{block_index}"
elif block[0] == "output":
module_pfx = f"lllite_unet_output_blocks_{block[1]}_1_transformer_blocks_{block_index}"
else:
raise Exception("invalid block name")
return module_pfx
def load_control_net_lllite_patch(path, cond_image, multiplier, num_steps, start_percent, end_percent):
# calculate start and end step
start_step = math.floor(num_steps * start_percent * 0.01) if start_percent > 0 else 0
end_step = math.floor(num_steps * end_percent * 0.01) if end_percent > 0 else num_steps
# load weights
ctrl_sd = comfy.utils.load_torch_file(path, safe_load=True)
# split each weights for each module
module_weights = {}
for key, value in ctrl_sd.items():
fragments = key.split(".")
module_name = fragments[0]
weight_name = ".".join(fragments[1:])
if module_name not in module_weights:
module_weights[module_name] = {}
module_weights[module_name][weight_name] = value
# load each module
modules = {}
for module_name, weights in module_weights.items():
# ここの自動判定を何とかしたい
if "conditioning1.4.weight" in weights:
depth = 3
elif weights["conditioning1.2.weight"].shape[-1] == 4:
depth = 2
else:
depth = 1
module = LLLiteModule(
name=module_name,
is_conv2d=weights["down.0.weight"].ndim == 4,
in_dim=weights["down.0.weight"].shape[1],
depth=depth,
cond_emb_dim=weights["conditioning1.0.weight"].shape[0] * 2,
mlp_dim=weights["down.0.weight"].shape[0],
multiplier=multiplier,
num_steps=num_steps,
start_step=start_step,
end_step=end_step,
)
info = module.load_state_dict(weights)
modules[module_name] = module
if len(modules) == 1:
module.is_first = True
print(f"loaded {path} successfully, {len(modules)} modules")
# cond imageをセットする
cond_image = cond_image.permute(0, 3, 1, 2) # b,h,w,3 -> b,3,h,w
cond_image = cond_image * 2.0 - 1.0 # 0-1 -> -1-+1
for module in modules.values():
module.set_cond_image(cond_image)
class control_net_lllite_patch:
def __init__(self, modules):
self.modules = modules
def __call__(self, q, k, v, extra_options):
module_pfx = extra_options_to_module_prefix(extra_options)
is_attn1 = q.shape[-1] == k.shape[-1] # self attention
if is_attn1:
module_pfx = module_pfx + "_attn1"
else:
module_pfx = module_pfx + "_attn2"
module_pfx_to_q = module_pfx + "_to_q"
module_pfx_to_k = module_pfx + "_to_k"
module_pfx_to_v = module_pfx + "_to_v"
if module_pfx_to_q in self.modules:
q = q + self.modules[module_pfx_to_q](q)
if module_pfx_to_k in self.modules:
k = k + self.modules[module_pfx_to_k](k)
if module_pfx_to_v in self.modules:
v = v + self.modules[module_pfx_to_v](v)
return q, k, v
def to(self, device):
for d in self.modules.keys():
self.modules[d] = self.modules[d].to(device)
return self
return control_net_lllite_patch(modules)
class LLLiteModule(torch.nn.Module):
def __init__(
self,
name: str,
is_conv2d: bool,
in_dim: int,
depth: int,
cond_emb_dim: int,
mlp_dim: int,
multiplier: int,
num_steps: int,
start_step: int,
end_step: int,
):
super().__init__()
self.name = name
self.is_conv2d = is_conv2d
self.multiplier = multiplier
self.num_steps = num_steps
self.start_step = start_step
self.end_step = end_step
self.is_first = False
modules = []
modules.append(torch.nn.Conv2d(3, cond_emb_dim // 2, kernel_size=4, stride=4, padding=0)) # to latent (from VAE) size*2
if depth == 1:
modules.append(torch.nn.ReLU(inplace=True))
modules.append(torch.nn.Conv2d(cond_emb_dim // 2, cond_emb_dim, kernel_size=2, stride=2, padding=0))
elif depth == 2:
modules.append(torch.nn.ReLU(inplace=True))
modules.append(torch.nn.Conv2d(cond_emb_dim // 2, cond_emb_dim, kernel_size=4, stride=4, padding=0))
elif depth == 3:
# kernel size 8は大きすぎるので、4にする / kernel size 8 is too large, so set it to 4
modules.append(torch.nn.ReLU(inplace=True))
modules.append(torch.nn.Conv2d(cond_emb_dim // 2, cond_emb_dim // 2, kernel_size=4, stride=4, padding=0))
modules.append(torch.nn.ReLU(inplace=True))
modules.append(torch.nn.Conv2d(cond_emb_dim // 2, cond_emb_dim, kernel_size=2, stride=2, padding=0))
self.conditioning1 = torch.nn.Sequential(*modules)
if self.is_conv2d:
self.down = torch.nn.Sequential(
torch.nn.Conv2d(in_dim, mlp_dim, kernel_size=1, stride=1, padding=0),
torch.nn.ReLU(inplace=True),
)
self.mid = torch.nn.Sequential(
torch.nn.Conv2d(mlp_dim + cond_emb_dim, mlp_dim, kernel_size=1, stride=1, padding=0),
torch.nn.ReLU(inplace=True),
)
self.up = torch.nn.Sequential(
torch.nn.Conv2d(mlp_dim, in_dim, kernel_size=1, stride=1, padding=0),
)
else:
self.down = torch.nn.Sequential(
torch.nn.Linear(in_dim, mlp_dim),
torch.nn.ReLU(inplace=True),
)
self.mid = torch.nn.Sequential(
torch.nn.Linear(mlp_dim + cond_emb_dim, mlp_dim),
torch.nn.ReLU(inplace=True),
)
self.up = torch.nn.Sequential(
torch.nn.Linear(mlp_dim, in_dim),
)
self.depth = depth
self.cond_image = None
self.cond_emb = None
self.current_step = 0
# @torch.inference_mode()
def set_cond_image(self, cond_image):
# print("set_cond_image", self.name)
self.cond_image = cond_image
self.cond_emb = None
self.current_step = 0
def forward(self, x):
if self.num_steps > 0:
if self.current_step < self.start_step:
self.current_step += 1
return torch.zeros_like(x)
elif self.current_step >= self.end_step:
if self.is_first and self.current_step == self.end_step:
print(f"end LLLite: step {self.current_step}")
self.current_step += 1
if self.current_step >= self.num_steps:
self.current_step = 0 # reset
return torch.zeros_like(x)
else:
if self.is_first and self.current_step == self.start_step:
print(f"start LLLite: step {self.current_step}")
self.current_step += 1
if self.current_step >= self.num_steps:
self.current_step = 0 # reset
if self.cond_emb is None:
# print(f"cond_emb is None, {self.name}")
cx = self.conditioning1(self.cond_image.to(x.device, dtype=x.dtype))
if not self.is_conv2d:
# reshape / b,c,h,w -> b,h*w,c
n, c, h, w = cx.shape
cx = cx.view(n, c, h * w).permute(0, 2, 1)
self.cond_emb = cx
cx = self.cond_emb
# print(f"forward {self.name}, {cx.shape}, {x.shape}")
# uncond/condでxはバッチサイズが2倍
if x.shape[0] != cx.shape[0]:
if self.is_conv2d:
cx = cx.repeat(x.shape[0] // cx.shape[0], 1, 1, 1)
else:
# print("x.shape[0] != cx.shape[0]", x.shape[0], cx.shape[0])
cx = cx.repeat(x.shape[0] // cx.shape[0], 1, 1)
cx = torch.cat([cx, self.down(x)], dim=1 if self.is_conv2d else 2)
cx = self.mid(cx)
cx = self.up(cx)
return cx * self.multiplier
class LLLiteLoader:
def __init__(self):
pass
@classmethod
def INPUT_TYPES(s):
return {
"required": {
"model": ("MODEL",),
"model_name": (get_file_list(folder_paths.get_folder_paths("controlnet")[0]),),
"cond_image": ("IMAGE",),
"strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),
"steps": ("INT", {"default": 0, "min": 0, "max": 200, "step": 1}),
"start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step": 0.1}),
"end_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 100.0, "step": 0.1}),
}
}
RETURN_TYPES = ("MODEL",)
FUNCTION = "load_lllite"
CATEGORY = "EasyUse/Loader"
def load_lllite(self, model, model_name, cond_image, strength, steps, start_percent, end_percent):
# cond_image is b,h,w,3, 0-1
model_path = os.path.join(folder_paths.get_folder_paths("controlnet")[0], model_name)
model_lllite = model.clone()
patch = load_control_net_lllite_patch(model_path, cond_image, strength, steps, start_percent, end_percent)
if patch is not None:
model_lllite.set_model_attn1_patch(patch)
model_lllite.set_model_attn2_patch(patch)
return (model_lllite,)
NODE_CLASS_MAPPINGS = {"easy LLLiteLoader": LLLiteLoader}
NODE_DISPLAY_NAME_MAPPINGS = {
"easy LLLiteLoader": "EasyLLLite",
}
+23 -23
View File
@@ -42,7 +42,7 @@ app.registerExtension({
link_info,
output
) {
console.log("onConnectionsChange");
// console.log("onConnectionsChange");
//On Disconnect
if (slotType == 1 && !isChangeConnect) {
this.inputs[slot].type = '*';
@@ -90,7 +90,7 @@ app.registerExtension({
}
this.clone = function () {
console.log("CLONE");
// console.log("CLONE");
const cloned = SetNode.prototype.clone.apply(this);
//cloned.inputs = [];
cloned.inputs[0].name = '*';
@@ -106,8 +106,8 @@ app.registerExtension({
this.update = function() {
console.log("SetNode.update()");
console.log(this.widgets[0].value);
// console.log("SetNode.update()");
// console.log(this.widgets[0].value);
if (node.graph) {
this.findGetters(node.graph).forEach((getter) => {
getter.setType(this.inputs[0].type);
@@ -146,9 +146,9 @@ app.registerExtension({
}
onRemoved() {
console.log("onRemove");
console.log(this);
console.log(this.flags);
// console.log("onRemove");
// console.log(this);
// console.log(this.flags);
const allGetters = this.graph._nodes.filter((otherNode) => otherNode.type == "easy getNode");
allGetters.forEach((otherNode) => {
if (otherNode.setComboValues) {
@@ -221,8 +221,8 @@ app.registerExtension({
this.setName = function(name) {
console.log("renaming getter: ");
console.log(node.widgets[0].value + " -> " + name);
// console.log("renaming getter: ");
// console.log(node.widgets[0].value + " -> " + name);
node.widgets[0].value = name;
node.onRename();
node.serialize();
@@ -230,7 +230,7 @@ app.registerExtension({
this.onRename = function() {
console.log("onRename");
// console.log("onRename");
const setter = this.findSetter(node.graph);
if (setter) {
@@ -248,13 +248,13 @@ app.registerExtension({
};
this.validateLinks = function() {
console.log("validating links");
// console.log("validating links");
if (this.outputs[0].type != '*' && this.outputs[0].links) {
console.log("in");
// console.log("in");
this.outputs[0].links.forEach((linkId) => {
const link = node.graph.links[linkId];
if (link && link.type != this.outputs[0].type && link.type != '*') {
console.log("removing link");
// console.log("removing link");
node.graph.removeLink(linkId)
}
})
@@ -286,10 +286,10 @@ app.registerExtension({
getInputLink(slot) {
console.log("get.getInputLink(): " + slot);
// console.log("get.getInputLink(): " + slot);
const setter = this.findSetter(this.graph);
console.log("setter:");
console.log(setter);
// console.log("setter:");
// console.log(setter);
// const setters = app.graph._nodes.filter((otherNode) => {
@@ -311,16 +311,16 @@ app.registerExtension({
if (setter) {
const slot_info = setter.inputs[slot];
console.log("slot info");
console.log(slot_info);
console.log(this.graph.links);
// console.log("slot info");
// console.log(slot_info);
// console.log(this.graph.links);
const link = this.graph.links[ slot_info.link ];
console.log("link:");
console.log(link);
// console.log("link:");
// console.log(link);
return link;
} else {
console.log(this.widgets[0]);
console.log(this.widgets[0].value);
// console.log(this.widgets[0]);
// console.log(this.widgets[0].value);
throw new Error("No setter found for " + this.widgets[0].value + "(" + this.type + ")");
}
+66
View File
@@ -0,0 +1,66 @@
import { app } from "../../../scripts/app.js";
app.registerExtension({
name: "comfy.easyUse.imageWidgets",
nodeCreated(node) {
if (["easy imageSize","easy imageSizeByLongerSide"].includes(node.comfyClass)) {
const inputEl = document.createElement("textarea");
inputEl.className = "comfy-multiline-input";
inputEl.readOnly = true
const widget = node.addDOMWidget("info", "customtext", inputEl, {
getValue() {
return inputEl.value;
},
setValue(v) {
inputEl.value = v;
},
serialize: false
});
widget.inputEl = inputEl;
inputEl.addEventListener("input", () => {
widget.callback?.(widget.value);
});
}
},
beforeRegisterNodeDef(nodeType, nodeData, app) {
if (["easy imageSize","easy imageSizeByLongerSide"].includes(nodeData.name)) {
function populate(arr_text) {
var text = '';
for (let i = 0; i < arr_text.length; i++){
text += arr_text[i];
}
if (this.widgets) {
const pos = this.widgets.findIndex((w) => w.name === "info");
if (pos !== -1 && this.widgets[pos]) {
const w = this.widgets[pos]
w.value = text;
}
}
requestAnimationFrame(() => {
const sz = this.computeSize();
if (sz[0] < this.size[0]) {
sz[0] = this.size[0];
}
if (sz[1] < this.size[1]) {
sz[1] = this.size[1];
}
this.onResize?.(sz);
app.graph.setDirtyCanvas(true, false);
});
}
// When the node is executed we will be sent the input text, display this in the widget
const onExecuted = nodeType.prototype.onExecuted;
nodeType.prototype.onExecuted = function (message) {
onExecuted?.apply(this, arguments);
populate.call(this, message.text);
};
}
}
})