diff --git a/README.md b/README.md index 2ea9a9a..eeb458f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,18 @@ # ComfyUI-InstanceDiffusion -ComfyUI nodes to use InstanceDiffusion +ComfyUI nodes to use InstanceDiffusion. + +Original research repo: https://github.com/frank-xwang/InstanceDiffusion/tree/main?tab=readme-ov-file + + +## Installation + +### Models +Download the three ckpt files from https://huggingface.co/spaces/logtd/instancediffusion/tree/main + +1. Place `fusers.ckpt` in `instance_models/fuser_models/` +2. Place `positionnet.ckpt` in `instance_models/positionnet_models/` +3. Place `scaleu.ckpt` in `instance_models/scaleu_models/` + +## Nodes + +### \ No newline at end of file diff --git a/instance_models/fuser_models/place_fusers_ckpt_here b/instance_models/fuser_models/place_fusers_ckpt_here new file mode 100644 index 0000000..e69de29 diff --git a/instance_models/positionnet_models/place_positionnet_ckpt_here b/instance_models/positionnet_models/place_positionnet_ckpt_here new file mode 100644 index 0000000..e69de29 diff --git a/instance_models/scaleu_models/place_scaleu_ckpt_here b/instance_models/scaleu_models/place_scaleu_ckpt_here new file mode 100644 index 0000000..e69de29 diff --git a/nodes/instance_diffusion_tracking_prompt_node.py b/nodes/instance_diffusion_tracking_prompt_node.py index 030d02e..e56d3e0 100644 --- a/nodes/instance_diffusion_tracking_prompt_node.py +++ b/nodes/instance_diffusion_tracking_prompt_node.py @@ -5,20 +5,23 @@ from ..conditioning.instance_conditioning import InstanceConditioning class InstanceDiffusionTrackingPromptNode: @classmethod def INPUT_TYPES(s): - return {"required": {"conditioning": ("CONDITIONING", ), + return {"required": {"positive": ("CONDITIONING", ), + "negative": ("CONDITIONING", ), "clip": ("CLIP", ), "tracking": ("TRACKING", ), "positionnet": ("POSITIONNET", ), "fusers": ("FUSERS", ), "fusers_batch_size": ("INT", {"default": 5, "min": 0, "max": 10000, "step": 8}), - "text": ("STRING", {"multiline": True}), + "positive_text": ("STRING", {"multiline": True}), + "negative_text": ("STRING", {"multiline": True}), }} - RETURN_TYPES = ("CONDITIONING",) + RETURN_TYPES = ("CONDITIONING", "CONDITIONING") + RETURN_NAMES = ("positive", "negative") FUNCTION = "append" CATEGORY = "instance/conditioning" - def append(self, conditioning, clip, tracking, fusers, positionnet, fusers_batch_size, text): + def _get_position_conds(self, clip, tracking, text): # Get prompts and their class id and trakcer id prompt_pairs = extract_prompts(text) @@ -38,6 +41,9 @@ class InstanceDiffusionTrackingPromptNode: 'positions': tracking[class_id][tracker_id]} position_conds.append(position_cond) + return position_conds + + def _apply_position_conds(self, position_conds, conditioning, fusers, positionnet, fusers_batch_size): # Add prompts+embeddings to the input conditionings cond_out = [] for t in conditioning: @@ -58,4 +64,18 @@ class InstanceDiffusionTrackingPromptNode: cond_out.append(n) - return (cond_out,) + return cond_out + + def append(self, positive, negative, clip, tracking, fusers, positionnet, fusers_batch_size, positive_text, negative_text): + + positive_positions = self._get_position_conds( + clip, tracking, positive_text) + positive = self._apply_position_conds( + positive_positions, positive, fusers, positionnet, fusers_batch_size) + + negative_positions = self._get_position_conds( + clip, tracking, negative_text) + negative = self._apply_position_conds( + negative_positions, negative, fusers, positionnet, fusers_batch_size) + + return (positive, negative)