add ImpactWildCards

refactoring directory structure
This commit is contained in:
Dr.Lt.Data
2023-06-15 10:53:24 +09:00
parent 6f7a503915
commit f9e002e023
21 changed files with 406 additions and 63 deletions
+2 -1
View File
@@ -1,2 +1,3 @@
__pycache__
*.ini
*.ini
wildcards/**
+4 -1
View File
@@ -64,7 +64,10 @@ This takes latent as input and outputs latent as the result.
* ImageSender, ImageReceiver - The images generated in ImageSender are automatically sent to the ImageReceiver with the same link_id.
* NOTE: requires this [patch](https://github.com/comfyanonymous/ComfyUI/pull/746)
* Switch (image,mask), Switch (latent) - Among multiple inputs, it selects the input designated by the selector and outputs it. The first input must be provided, while the others are optional. However, if the input specified by the selector is not connected, an error may occur.
* ImpactWildcardProcessor - The text is generated by processing the wildcard in the Text. If the mode is set to "populate", a dynamic prompt is generated with each execution and the input is filled in the second textbox. If the mode is set to "fixed", the content of the second textbox remains unchanged.
* When an image is generated with the "fixed" mode, the prompt used for that particular generation is stored in the metadata.
* Known Issue: The presetText.js script from **pythongosssss's [ComfyUI-Custom-Scripts](https://github.com/pythongosssss/ComfyUI-Custom-Scripts)** is causing a conflict, preventing it from being used together.
# Feature
* Interactive SAM Detector (Clipspace) - When you right-click on a node that has 'MASK' and 'IMAGE' outputs, a context menu will open. From this menu, you can either open a dialog to create a SAM Mask using 'Open in SAM Detector', or copy the content (likely mask data) using 'Copy (Clipspace)' and generate a mask using 'Impact SAM Detector' from the clipspace menu, and then paste it using 'Paste (Clipspace)'.
+37 -19
View File
@@ -2,18 +2,27 @@ import shutil
import folder_paths
import os
import sys
import importlib
comfy_path = os.path.dirname(folder_paths.__file__)
impact_path = os.path.dirname(__file__)
impact_path = os.path.join(os.path.dirname(__file__))
modules_path = os.path.join(os.path.dirname(__file__), "modules")
wildcards_path = os.path.join(os.path.dirname(__file__), "wildcards")
sys.path.append(impact_path)
sys.path.append(modules_path)
import impact_config
print(f"### Loading: ComfyUI-Impact-Pack ({impact_config.version})")
import impact.config
print(f"### Loading: ComfyUI-Impact-Pack ({impact.config.version})")
def do_install():
spec = importlib.util.spec_from_file_location('impact_install', os.path.join(os.path.dirname(__file__), 'install.py'))
impact_install = importlib.util.module_from_spec(spec)
spec.loader.exec_module(impact_install)
# ensure dependency
if impact_config.read_config()[1] < impact_config.dependency_version:
import install # to install dependencies
if impact.config.read_config()[1] < impact.config.dependency_version:
do_install()
# Core
# recheck dependencies for colab
try:
@@ -31,10 +40,11 @@ try:
from skimage.measure import label, regionprops
from collections import namedtuple
except:
import importlib
print("### ComfyUI-Impact-Pack: Reinstall dependencies (several dependencies are missing.)")
import install
do_install()
import impact_server # to load server api
import impact.impact_server # to load server api
def setup_js():
# remove garbage
@@ -55,10 +65,12 @@ def setup_js():
setup_js()
import legacy_nodes
from impact_pack import *
from detectors import *
from impact_pipe import *
import impact.legacy_nodes
from impact.impact_pack import *
from impact.detectors import *
from impact.pipe import *
impact.wildcards.read_wildcard_dict(wildcards_path)
NODE_CLASS_MAPPINGS = {
"SAMLoader": SAMLoader,
@@ -128,13 +140,19 @@ NODE_CLASS_MAPPINGS = {
"ImageMaskSwitch": ImageMaskSwitch,
"LatentSwitch": LatentSwitch,
"MaskPainter": legacy_nodes.MaskPainter,
"MMDetLoader": legacy_nodes.MMDetLoader,
"SegsMaskCombine": legacy_nodes.SegsMaskCombine,
"BboxDetectorForEach": legacy_nodes.BboxDetectorForEach,
"SegmDetectorForEach": legacy_nodes.SegmDetectorForEach,
"BboxDetectorCombined": legacy_nodes.BboxDetectorCombined,
"SegmDetectorCombined": legacy_nodes.SegmDetectorCombined,
# "SaveConditioning": SaveConditioning,
# "LoadConditioning": LoadConditioning,
"ImpactWildcardProcessor": ImpactWildcardProcessor,
"ImpactLogger": ImpactLogger,
"MaskPainter": impact.legacy_nodes.MaskPainter,
"MMDetLoader": impact.legacy_nodes.MMDetLoader,
"SegsMaskCombine": impact.legacy_nodes.SegsMaskCombine,
"BboxDetectorForEach": impact.legacy_nodes.BboxDetectorForEach,
"SegmDetectorForEach": impact.legacy_nodes.SegmDetectorForEach,
"BboxDetectorCombined": impact.legacy_nodes.BboxDetectorCombined,
"SegmDetectorCombined": impact.legacy_nodes.SegmDetectorCombined,
}
NODE_DISPLAY_NAME_MAPPINGS = {
+4 -5
View File
@@ -4,16 +4,15 @@ import subprocess
comfy_path = '../..'
if sys.argv[0] == 'install.py':
sys.path.append('.') # for portable version
sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "comfy"))
sys.path.append('.') # for portable version
sys.path.append(comfy_path)
import platform
import folder_paths
from torchvision.datasets.utils import download_url
import impact_config
import impact.config
print("### ComfyUI-Impact-Pack: Check dependencies")
@@ -118,7 +117,7 @@ def install():
print(f"### ComfyUI-Impact-Pack: onnx model directory created ({onnx_path})")
os.mkdir(onnx_path)
impact_config.write_config(comfy_path)
impact.config.write_config(comfy_path)
install()
+62 -7
View File
@@ -153,6 +153,42 @@ app.registerExtension({
ComfyApp.open_maskeditor();
});
}
if(node.comfyClass == "ImpactWildcardProcessor") {
let force_serializeValue = async (n,i) =>
{
if(n.widgets_values[2] == "Fixed") {
return node.widgets[1].value;
}
else {
let response = await fetch(`/impact/wildcards`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({text: n.widgets_values[0]})
});
let populated = await response.json();
n.widgets_values[2] = "Fixed";
n.widgets_values[1] = populated.text;
node.widgets[1].value = populated.text;
return populated.text;
}
};
// prevent hooking by dynamicPrompt.js
Object.defineProperty(node.widgets[0], "serializeValue", {
set: () => {},
get: (value) => { return (n,i) => { return n.widgets_values[i]; }; }
});
Object.defineProperty(node.widgets[1], "serializeValue", {
set: () => {},
get: (value) => { return force_serializeValue; }
});
}
if (node.comfyClass == "PreviewBridge" || node.comfyClass == "MaskPainter") {
node.widgets[0].value = '#placeholder';
@@ -169,13 +205,31 @@ app.registerExtension({
node.widgets[0].value = {...input_tracking[id][1]};
input_dirty[id] = false;
need_invalidate = true
this._images = app.nodeOutputs[id].images;
}
node.widgets[0].value['image_hash'] = app.nodeOutputs[id]['aux'][0];
node.widgets[0].value['forward_filename'] = app.nodeOutputs[id]['aux'][1][0]['filename'];
node.widgets[0].value['forward_subfolder'] = app.nodeOutputs[id]['aux'][1][0]['subfolder'];
node.widgets[0].value['forward_type'] = app.nodeOutputs[id]['aux'][1][0]['type'];
app.nodeOutputs[id].images = [node.widgets[0].value];
let filename = app.nodeOutputs[id]['aux'][1][0]['filename'];
let subfolder = app.nodeOutputs[id]['aux'][1][0]['subfolder'];
let type = app.nodeOutputs[id]['aux'][1][0]['type'];
let item =
{
image_hash: app.nodeOutputs[id]['aux'][0],
forward_filename: app.nodeOutputs[id]['aux'][1][0]['filename'],
forward_subfolder: app.nodeOutputs[id]['aux'][1][0]['subfolder'],
forward_type: app.nodeOutputs[id]['aux'][1][0]['type']
};
app.nodeOutputs[id].images = [{
...node._images[0],
...item
}];
node.widgets[0].value =
{
...node._images[0],
...item
};
if(need_invalidate) {
Promise.all(
@@ -184,8 +238,7 @@ app.registerExtension({
const img = new Image();
img.onload = () => r(img);
img.onerror = () => r(null);
img.src = "/view?" + new URLSearchParams(src[0]).toString();
console.log(`new img => ${img.src}`);
img.src = "/view?" + new URLSearchParams(src).toString();
});
})
).then((imgs) => {
@@ -193,6 +246,8 @@ app.registerExtension({
this.setSizeForImage?.();
app.graph.setDirtyCanvas(true);
});
app.nodeOutputs[id].images[0] = { ...node.widgets[0].value };
}
return app.nodeOutputs[id].images;
+13 -3
View File
@@ -575,16 +575,26 @@ class ImpactSamEditorDialog extends ComfyDialog {
const index = ComfyApp.clipspace.widgets.findIndex(obj => obj.name === 'image');
if(index >= 0)
ComfyApp.clipspace.widgets[index].value = item;
ComfyApp.clipspace.widgets[index].value = `${filename} [temp]`;
}
const dataURL = save_canvas.toDataURL();
const blob = dataURLToBlob(dataURL);
const original_blob = loadedImageToBlob(this.image);
let original_url = new URL(this.image.src);
const original_ref = { filename: original_url.searchParams.get('filename') };
let original_subfolder = original_url.searchParams.get("subfolder");
if(original_subfolder)
original_ref.subfolder = original_subfolder;
let original_type = original_url.searchParams.get("type");
if(original_type)
original_ref.type = original_type;
formData.append('image', blob, filename);
formData.append('original_image', original_blob);
formData.append('original_ref', JSON.stringify(original_ref));
formData.append('type', "temp");
await uploadMask(item, formData);
@@ -1,7 +1,7 @@
import configparser
import os
version = "V2.10.2"
version = "V2.12.3"
dependency_version = 1
+4 -7
View File
@@ -1,20 +1,15 @@
import os
import sys
import mmcv
from mmdet.apis import (inference_detector, init_detector)
from mmdet.evaluation import get_classes
from segment_anything import SamPredictor
import torch.nn.functional as F
from impact_utils import *
from impact.utils import *
from collections import namedtuple
import numpy as np
from skimage.measure import label, regionprops
main_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
sys.path.append(os.path.dirname(__file__))
sys.path.append(main_dir)
import nodes
import comfy_extras.nodes_upscale_model as model_upscale
from server import PromptServer
@@ -290,6 +285,8 @@ def make_sam_mask(sam_model, segs, image, detection_hint, dilation,
predictor = SamPredictor(sam_model)
image = np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8)
print(f"image.shape: {image.shape}")
predictor.set_image(image, "RGB")
total_masks = []
@@ -481,7 +478,7 @@ class ONNXDetector(BBoxDetector):
def detect(self, image, threshold, dilation, crop_factor, drop_size=1):
drop_size = max(drop_size, 1)
try:
import onnx
import impact.onnx as onnx
h = image.shape[1]
w = image.shape[2]
+2 -2
View File
@@ -1,5 +1,5 @@
import impact_core as core
from impact_config import MAX_RESOLUTION
import impact.core as core
from impact.config import MAX_RESOLUTION
class SAMDetectorCombined:
+177 -10
View File
@@ -6,10 +6,15 @@ import comfy.sd
import warnings
from segment_anything import sam_model_registry
from impact_utils import *
import impact_core as core
from impact_core import SEG, NO_BBOX_DETECTOR, NO_SEGM_DETECTOR
from impact_config import MAX_RESOLUTION
from impact.utils import *
import impact.core as core
from impact.core import SEG, NO_BBOX_DETECTOR, NO_SEGM_DETECTOR
from impact.config import MAX_RESOLUTION
from PIL import Image
import numpy as np
import hashlib
import json
import safetensors.torch
warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
@@ -1203,12 +1208,29 @@ class SubtractMask:
import nodes
def get_image_hash(arr):
split_index1 = arr.shape[0] // 2
split_index2 = arr.shape[1] // 2
part1 = arr[:split_index1, :split_index2]
part2 = arr[:split_index1, split_index2:]
part3 = arr[split_index1:, :split_index2]
part4 = arr[split_index1:, split_index2:]
# 각 부분을 합산
sum1 = np.sum(part1)
sum2 = np.sum(part2)
sum3 = np.sum(part3)
sum4 = np.sum(part4)
return hash((sum1, sum2, sum3, sum4))
preview_hash_map = {}
class PreviewBridge(nodes.PreviewImage):
@classmethod
def INPUT_TYPES(s):
return {"required": {"images": ("IMAGE",), },
"hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO", },
"hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO", "unique_id": "UNIQUE_ID"},
"optional": {"image": (["#placeholder"], )},
}
@@ -1218,8 +1240,23 @@ class PreviewBridge(nodes.PreviewImage):
CATEGORY = "ImpactPack/Util"
def doit(self, images, image, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None):
if image == "#placeholder" or image['image_hash'] != id(images) or ('0' in image and 'forward_filename' not in image[0]):
def doit(self, images, image, filename_prefix="ComfyUI", prompt=None, extra_pnginfo=None, unique_id=None):
global preview_hash_map
if image != "#placeholder" and isinstance(image, str):
image_path = folder_paths.get_annotated_filepath(image)
img = Image.open(image_path).convert("RGB")
data = np.array(img)
image_hash = get_image_hash(data)
else:
data = (255. * images[0].cpu().numpy()).astype(int)
image_hash = get_image_hash(data)
is_changed = False
if unique_id not in preview_hash_map or preview_hash_map[unique_id] != image_hash:
preview_hash_map[unique_id] = image_hash
is_changed = True
if is_changed or image == "#placeholder":
# new input image
res = self.save_images(images, filename_prefix, prompt, extra_pnginfo)
@@ -1232,7 +1269,7 @@ class PreviewBridge(nodes.PreviewImage):
image, mask = nodes.LoadImage().load_image(filepath)
res['ui']['aux'] = [id(images), res['ui']['images']]
res['ui']['aux'] = [image_hash, res['ui']['images']]
res['result'] = (image, mask, )
return res
@@ -1257,7 +1294,7 @@ class PreviewBridge(nodes.PreviewImage):
if 'type' in image and image['type'] != "":
imgpath += f" [{image['type']}]"
res['ui']['aux'] = [id(images), [forward]]
res['ui']['aux'] = [image_hash, [forward]]
res['result'] = nodes.LoadImage().load_image(imgpath)
return res
@@ -1381,4 +1418,134 @@ class LatentSwitch:
elif select == 3:
return (latent3_opt,)
else:
return (latent4_opt,)
return (latent4_opt,)
class SaveConditioning:
def __init__(self):
self.output_dir = folder_paths.get_output_directory()
@classmethod
def INPUT_TYPES(s):
return {"required": {"conditioning": ("CONDITIONING", ),
"filename_prefix": ("STRING", {"default": "conditioning/ComfyUI"}),
},
"hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
}
RETURN_TYPES = ()
FUNCTION = "doit"
OUTPUT_NODE = True
CATEGORY = "_for_testing"
def doit(self, conditioning, filename_prefix, prompt=None, extra_pnginfo=None):
# support save metadata for latent sharing
prompt_info = ""
if prompt is not None:
prompt_info = json.dumps(prompt)
for tensor_data, meta_data in conditioning:
full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir)
metadata = {"prompt": prompt_info}
if extra_pnginfo is not None:
for x in extra_pnginfo:
metadata[x] = json.dumps(extra_pnginfo[x])
file = f"{filename}_{counter:05}_.conditioning"
file = os.path.join(full_output_folder, file)
print(f"meta_data:{meta_data}")
print(f"tensor_data:{tensor_data}")
output = {"conditioning": tensor_data}
metadata['conditioning_aux'] = json.dumps(meta_data)
safetensors.torch.save_file(output, file, metadata=metadata)
return {}
class LoadConditioning:
@classmethod
def INPUT_TYPES(s):
input_dir = folder_paths.get_input_directory()
files = [f for f in os.listdir(input_dir) if os.path.isfile(os.path.join(input_dir, f)) and f.endswith(".conditioning")]
return {"required": {"conditioning": [sorted(files), ]}, }
CATEGORY = "_for_testing"
RETURN_TYPES = ("CONDITIONING", )
FUNCTION = "load"
def load(self, conditioning):
conditioning_path = folder_paths.get_annotated_filepath(conditioning)
data = safetensors.torch.load_file(conditioning_path, device="cpu")
return ([[data['conditioning'], {}]], )
@classmethod
def IS_CHANGED(s, conditioning):
image_path = folder_paths.get_annotated_filepath(conditioning)
m = hashlib.sha256()
with open(image_path, 'rb') as f:
m.update(f.read())
return m.digest().hex()
@classmethod
def VALIDATE_INPUTS(s, conditioning):
if not folder_paths.exists_annotated_filepath(conditioning):
return "Invalid conditioning file: {}".format(conditioning)
return True
class ImpactWildcardProcessor:
@classmethod
def INPUT_TYPES(s):
return {"required": {
"wildcard_text": ("STRING", {"multiline": True}),
"populated_text": ("STRING", {"multiline": True}),
"mode": (["Populate", "Fixed"], ),
},
}
CATEGORY = "ImpactPack/Prompt"
RETURN_TYPES = ("STRING", )
FUNCTION = "doit"
def doit(self, wildcard_text, populated_text, mode):
return (populated_text, )
class ImpactLogger:
@classmethod
def INPUT_TYPES(s):
return {"required": {
"text": ("STRING", {"default": ""}),
},
"hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"},
}
CATEGORY = "ImpactPack/Debug"
OUTPUT_NODE = True
RETURN_TYPES = ()
FUNCTION = "doit"
def doit(self, text, prompt, extra_pnginfo):
print(f"[IMPACT LOGGER]: {text}")
print(f" PROMPT: {prompt}")
# for x in prompt:
# if 'inputs' in x and 'populated_text' in x['inputs']:
# print(f"PROMP: {x['10']['inputs']['populated_text']}")
#
# for x in extra_pnginfo['workflow']['nodes']:
# if x['type'] == 'ImpactWildcardProcessor':
# print(f" WV : {x['widgets_values'][1]}\n")
return {}
@@ -5,13 +5,14 @@ from aiohttp import web
import server
import folder_paths
import impact_core as core
import impact_pack
import impact.core as core
import impact.impact_pack as impact_pack
from segment_anything import SamPredictor, sam_model_registry
import numpy as np
import nodes
from PIL import Image
import io
import impact.wildcards as wildcards
@server.PromptServer.instance.routes.post("/upload/temp")
async def upload_image(request):
@@ -93,12 +94,15 @@ async def load_sam_model(request):
sam_predictor.set_image(image, "RGB")
print(f"ComfyUI-Impact-Pack: SAM model loaded. ")
@server.PromptServer.instance.routes.post("/sam/release")
async def release_sam(request):
global sam_predictor
with sam_lock:
del sam_predictor
sam_predictor = None
print(f"ComfyUI-Impact-Pack: unloading SAM model")
@@ -146,3 +150,10 @@ async def sam_detect(request):
else:
return web.Response(status=400)
@server.PromptServer.instance.routes.post("/impact/wildcards")
async def populate_wildcards(request):
data = await request.json()
populated = wildcards.process(data['text'])
return web.json_response({"text": populated})
@@ -1,9 +1,9 @@
import folder_paths
import impact_core as core
from impact_utils import *
from impact_core import SEG
import impact.core as core
from impact.utils import *
from impact.core import SEG
import nodes
import os
class NO_BBOX_MODEL:
pass
+1 -1
View File
@@ -1,5 +1,5 @@
import additional_dependencies
from impact_utils import *
from impact.utils import *
additional_dependencies.ensure_onnx_package()
@@ -5,6 +5,7 @@ from PIL import Image, ImageFilter
LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.LANCZOS)
def pil2tensor(image):
return torch.from_numpy(np.array(image).astype(np.float32) / 255.0).unsqueeze(0)
+63
View File
@@ -0,0 +1,63 @@
import re
import random
import os
wildcard_dict = {}
def read_wildcard_dict(wildcard_path):
global wildcard_dict
for root, directories, files in os.walk(wildcard_path):
for file in files:
if file.endswith('.txt'):
file_path = os.path.join(root, file)
key = os.path.splitext(file)[0]
with open(file_path, 'r') as f:
lines = f.read().splitlines()
wildcard_dict[key] = lines
return wildcard_dict
def process(text):
def replace_options(string):
replacements_found = False
def replace_option(match):
nonlocal replacements_found
options = match.group(1).split('|')
replacement = random.choice(options)
replacements_found = True
return replacement
pattern = r'{([^{}]*?)}'
replaced_string = re.sub(pattern, replace_option, string)
return replaced_string, replacements_found
def replace_wildcard(string):
global wildcard_dict
pattern = r"__([\w.-]+)__"
matches = re.findall(pattern, string)
for match in matches:
if match in wildcard_dict:
replacement = random.choice(wildcard_dict[match])
string = string.replace(f"__{match}__", replacement, 1)
return string
# phase1: replace options
phase1, is_replaced = replace_options(text)
while is_replaced:
phase1, is_replaced = replace_options(phase1)
# phase2: replace wildcards
phase2 = replace_wildcard(phase1)
return phase2
View File
+9
View File
@@ -0,0 +1,9 @@
rose
orchid
iris
carnation
lily
daisy
chrysanthemum
daffodil
dahlia
+9
View File
@@ -0,0 +1,9 @@
diamond
emerald
sapphire
opal
ruby
topaz
pearl
rubyamethyst
aquamarine