Merge pull request #11 from Limitex/main

sync

README.md

@@ -4,15 +4,30 @@ This repository is a custom node in ComfyUI.
 
 ## Overview
 
+### Workflow 1
+
 This is a program that allows you to use Huggingface Diffusers module with ComfyUI. Additionally, Stream Diffusion is also available.
 
 ![Workflow](img/workflow.png)
 
+### Workflow 2
+
 In addition, real-time generation is possible by doing the following.
 When running, please enable Auto Queue in Extra options.
 
 ![Workflow2](img/workflow2.png)
 
+### Workflow 3
+
+In combination with [VideoHelperSuite](https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git), you can also run vid2vid.
+
+![Workflow3](img/workflow3.png)
+
+The execution looks like this:
+
+![Workflow3demo](img/workflow3demo.gif)
+
+
 ## Usage
 
 Run the following command inside ComfyUI/custom_nodes.
@@ -24,6 +39,9 @@ pip install -r requirements.txt
 git clone https://github.com/cumulo-autumn/StreamDiffusion.git
 python -m streamdiffusion.tools.install-tensorrt
 ```
+## Recommended Custom Nodes
+
+https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git
 
 ## Node
 

nodes.py

@@ -2,7 +2,7 @@ import copy
 import os
 import torch
 from safetensors.torch import load_file
-from .utils import SCHEDULERS, convert_images_to_tensors, token_auto_concat_embeds, vae_pt_to_vae_diffuser
+from .utils import SCHEDULERS, token_auto_concat_embeds, vae_pt_to_vae_diffuser, convert_images_to_tensors, convert_tensors_to_images, resize_images
 from comfy.model_management import get_torch_device
 import folder_paths
 from streamdiffusion import StreamDiffusion
@@ -294,6 +294,9 @@ class StreamDiffusionSampler:
                 "num": ("INT", {"default": 1, "min": 1, "max": 10000}),
                 "warmup": ("INT", {"default": 1, "min": 0, "max": 10000}),
             },
+            "optional" : {
+                "image" : ("IMAGE", )
+            }
         }
 
     RETURN_TYPES = ("IMAGE",)
@@ -302,7 +305,7 @@ class StreamDiffusionSampler:
 
     CATEGORY = "Diffusers/StreamDiffusion"
 
-    def sample(self, stream: StreamDiffusion, positive, negative, steps, cfg, delta, seed, num, warmup):
+    def sample(self, stream: StreamDiffusion, positive, negative, steps, cfg, delta, seed, num, warmup, image = None):
         stream.prepare(
             prompt = positive,
             negative_prompt = negative,
@@ -312,13 +315,25 @@ class StreamDiffusionSampler:
             seed = seed
         )
         
+        if image != None:
+            image = convert_tensors_to_images(image)
+            image = resize_images(image, (stream.width, stream.height))
+            
         for _ in range(warmup):
             stream()
 
         result = []
         for _ in range(num):
-            x_output = stream.txt2img()
-            result.append(postprocess_image(x_output, output_type="pil")[0])
+            x_outputs = []
+            if image is None:
+                x_outputs.append(stream.txt2img())
+            else:
+                stream(image[0])
+                for i in image[1:] + image[-1:]:
+                    x_outputs.append(stream(i))
+            for x_output in x_outputs:
+                result.append(postprocess_image(x_output, output_type="pil")[0])
+        
         return (convert_images_to_tensors(result),)
 
 class StreamDiffusionWarmup:

utils.py

@@ -2,6 +2,7 @@ import io
 import torch
 import requests
 import numpy as np
+from PIL import Image
 from omegaconf import OmegaConf
 from torchvision.transforms import ToTensor
 from diffusers.pipelines.stable_diffusion.convert_from_ckpt import (
@@ -202,5 +203,11 @@ def vae_pt_to_vae_diffuser(
     vae.save_pretrained(output_path)
 
 
-def convert_images_to_tensors(images):
-    return torch.stack([np.transpose(ToTensor()(image), (1, 2, 0)) for image in images])
+def convert_images_to_tensors(images: list[Image.Image]):
+    return torch.stack([np.transpose(ToTensor()(image), (1, 2, 0)) for image in images])
+
+def convert_tensors_to_images(images: torch.tensor):
+    return [Image.fromarray(np.clip(255. * image.cpu().numpy(), 0, 255).astype(np.uint8)) for image in images]
+
+def resize_images(images: list[Image.Image], size: tuple[int, int]):
+    return [image.resize(size) for image in images]