diff --git a/LBM_Relighting.py b/LBM_Relighting.py new file mode 100644 index 0000000..91973c5 --- /dev/null +++ b/LBM_Relighting.py @@ -0,0 +1,245 @@ +import os +import torch +from tqdm import tqdm +import requests +import shutil + +import folder_paths +import comfy.model_management as mm +from comfy.utils import load_torch_file + +from .lbm.models.lbm import LBMModel +from .lbm.models.unets import DiffusersUNet2DCondWrapper +from .lbm.models.vae import AutoencoderKLDiffusers +from .lbm.models.embedders import ConditionerWrapper +from diffusers.models import AutoencoderKL +from diffusers import FlowMatchEulerDiscreteScheduler + +class LBM_Relighting: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": (folder_paths.get_filename_list("diffusion_models"), {"default": "LBM_relighting.safetensors", "tooltip": "LBM model file loaded from 'ComfyUI/models/diffusion_models' folder"}), + "image": ("IMAGE",), + "steps": ("INT", {"default": 20, "min": 1, "max": 100, "tooltip": "LBM can achieve good results with just 20 step, but more steps can potentially improve quality"}), + "precision": (["fp32", "bf16", "fp16"], {"default": "bf16", "tooltip": "The official model was trained with bf16 precision"}), + }, + "optional": { + "bridge_noise_sigma": ("FLOAT", {"default": 0.005, "min": 0.0, "max": 0.1, "step": 0.001, "tooltip": "Controls the noise added in bridge matching process. Default: 0.005"}), + "max_samples": ("INT", {"default": 1, "min": 1, "max": 8, "tooltip": "Number of samples to generate in a batch"}) + } + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "process" + CATEGORY = 'πŸ”†LBM' + + def process(self, model, image, steps, precision, bridge_noise_sigma=0.005, max_samples=1): + model_path = self.ensure_model_exists(model) + + dtype_map = { + "bf16": torch.bfloat16, + "fp16": torch.float16, + "fp32": torch.float32 + } + base_dtype = dtype_map[precision] + + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + print(f"Loading LBM model...") + lbm_model = self.create_lbm_model(base_dtype, bridge_noise_sigma) + + sd = load_torch_file(model_path, device=offload_device, safe_load=True) + param_count = sum(1 for _ in lbm_model.named_parameters()) + for name, param in tqdm(lbm_model.named_parameters(), + desc=f"Loading model parameters", + total=param_count, + leave=True): + if name in sd: + param.data = sd[name].to(dtype=base_dtype) + + mm.soft_empty_cache() + + input_image = image.clone().permute(0, 3, 1, 2).to(device, base_dtype) * 2 - 1 + batch = {"source_image": input_image} + + lbm_model.vae.to(device) + z_source = lbm_model.vae.encode(batch[lbm_model.source_key]) + lbm_model.vae.cpu() + + lbm_model.to(device) + + result = lbm_model.sample( + z=z_source, + num_steps=steps, + conditioner_inputs=batch, + max_samples=max_samples, + ).clamp(-1, 1) + + out = result.permute(0, 2, 3, 1).cpu().float() + out = (out + 1) / 2 + + lbm_model.cpu() + mm.soft_empty_cache() + + return (out,) + + def ensure_model_exists(self, model_name): + model_paths = folder_paths.get_folder_paths("diffusion_models") + + if not model_paths: + raise RuntimeError("No diffusion_models paths found") + + for path in model_paths: + model_path = os.path.join(path, model_name) + if os.path.exists(model_path): + print(f"Model {model_name} found at {model_path}") + return model_path + + if model_name != "LBM_relighting.safetensors": + default_path = os.path.join(path, "LBM_relighting.safetensors") + if os.path.exists(default_path): + print(f"Default model found at {default_path}") + return default_path + + download_path = model_paths[0] + print(f"Model not found in any path. Downloading to {download_path}...") + + os.makedirs(download_path, exist_ok=True) + + model_url = "https://huggingface.co/jasperai/LBM_relighting/resolve/main/model.safetensors" + target_path = os.path.join(download_path, "LBM_relighting.safetensors") + temp_file = os.path.join(download_path, "temp_download.safetensors") + + try: + with requests.get(model_url, stream=True) as r: + r.raise_for_status() + total_size = int(r.headers.get('content-length', 0)) + + with open(temp_file, 'wb') as f, tqdm( + desc="Downloading LBM model", + total=total_size, + unit='B', + unit_scale=True, + unit_divisor=1024, + ) as pbar: + for chunk in r.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + pbar.update(len(chunk)) + + shutil.move(temp_file, target_path) + print(f"Model downloaded and saved as {target_path}") + return target_path + + except Exception as e: + if os.path.exists(temp_file): + os.remove(temp_file) + print(f"Error downloading model: {e}") + raise RuntimeError(f"Failed to download model: {e}") + + def create_lbm_model(self, dtype, bridge_noise_sigma=0.005): + config = { + "source_key": "source_image", + "target_key": "source_image", + "timestep_sampling": "custom_timesteps", + "selected_timesteps": [250, 500, 750, 1000], + "prob": [0.25, 0.25, 0.25, 0.25], + "bridge_noise_sigma": bridge_noise_sigma, + } + + denoiser = DiffusersUNet2DCondWrapper( + in_channels=4, + out_channels=4, + center_input_sample=False, + flip_sin_to_cos=True, + freq_shift=0, + down_block_types=[ + "DownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + ], + mid_block_type="UNetMidBlock2DCrossAttn", + up_block_types=["CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"], + only_cross_attention=False, + block_out_channels=[320, 640, 1280], + layers_per_block=2, + downsample_padding=1, + mid_block_scale_factor=1, + dropout=0.0, + act_fn="silu", + norm_num_groups=32, + norm_eps=1e-05, + cross_attention_dim=[320, 640, 1280], + transformer_layers_per_block=[1, 2, 10], + attention_head_dim=[5, 10, 20], + use_linear_projection=True, + time_embedding_type="positional", + ).to(dtype) + + conditioner = ConditionerWrapper(conditioners=[]) + + vae_config = { + "_class_name": "AutoencoderKL", + "_diffusers_version": "0.20.0.dev0", + "act_fn": "silu", + "block_out_channels": [128, 256, 512, 512], + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D" + ], + "force_upcast": True, + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 2, + "norm_num_groups": 32, + "out_channels": 3, + "sample_size": 1024, + "scaling_factor": 0.13025, + "up_block_types": [ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D" + ] + } + + vae = AutoencoderKLDiffusers(AutoencoderKL.from_config(vae_config)) + vae.freeze() + vae.to(dtype) + + scheduler_config = { + 'num_train_timesteps': 1000, + 'shift': 1.0, + 'use_dynamic_shifting': False, + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.012, + 'timestep_spacing': 'leading', + } + sampling_noise_scheduler = FlowMatchEulerDiscreteScheduler.from_config(scheduler_config) + + from .lbm.models.lbm import LBMConfig + lbm_config = LBMConfig(**config) + model = LBMModel( + lbm_config, + denoiser=denoiser, + sampling_noise_scheduler=sampling_noise_scheduler, + vae=vae, + conditioner=conditioner, + ).to(dtype) + + return model + + +NODE_CLASS_MAPPINGS = { + "LBM_Relighting": LBM_Relighting, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "LBM_Relighting": "Relighting (LBM)", +} \ No newline at end of file diff --git a/README.md b/README.md index 6082408..b69b5e6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,108 @@ -# ComfyUI-LBM -A ComfyUI custom node for Latent Bridge Matching (LBM), for fast image relighting processing. +# ComfyUI-LBM + +A ComfyUI implementation of Latent Bridge Matching (LBM) for efficient image relighting. This node utilizes the LBM algorithm to perform single-step image-to-image translation specifically for relighting tasks. + +![LBM-Relighting](example_workflows/LBM-RElighting.png) + +## Features + +- Fast image relighting with a single inference step +- Simplified workflow with just one node +- Optimized memory usage +- **Automatic model download** - the model will be downloaded automatically and properly renamed on first use +- Extensible architecture - support for depth and normal map processing coming soon + +## Installation + +1. Clone this repository to your `ComfyUI/custom_nodes` directory: +```bash +cd ComfyUI/custom_nodes +git clone https://github.com/1038lab/ComfyUI-LBM.git +``` + +2. Install the required dependencies: +```bash +cd ComfyUI/custom_nodes/ComfyUI-LBM +pip install -r requirements.txt +``` + +## Download Models + +The model will be automatically downloaded and renamed on first use, or you can manually download it: + +| Model | Description | Link | +| ----- | ----------- | ---- | +| LBM Relighting | Main model for image relighting | [Download](https://huggingface.co/jasperai/LBM_relighting/resolve/main/model.safetensors) | + +After downloading, place the model file in your `ComfyUI/models/diffusion_models` directory and rename it to `LBM_relighting.safetensors` + +## Basic Usage + +1. Add the "LBM Relighting" node from the `πŸ§ͺAILab/πŸ”†LBM` category +2. Connect an image source to the "LBM Relighting" node +3. Select the model file (defaults to `LBM_relighting.safetensors`) +4. Adjust the steps parameter as needed (default: 30) +5. Run the workflow + +### Parameters + +| Parameter | Description | Recommendation | +| --------- | ----------- | -------------- | +| **Model** | The LBM model file to use | Default is `LBM_relighting.safetensors` | +| **Steps** | Number of inference steps | Default is 30. More steps may improve quality at the cost of processing time | + +## Setting Tips + +| Setting | Recommendation | +| ------- | -------------- | +| **Steps** | For most images, 20-30 steps provides a good balance between quality and speed | +| **Input Resolution** | The model works best with images of 512x512 or higher resolution | +| **Memory Usage** | If you encounter memory issues, try processing images at a lower resolution | +| **Performance** | For batch processing, consider reducing steps to 15-20 for faster throughput | + +## About Model + +This implementation uses the Latent Bridge Matching (LBM) method from the paper "LBM: Latent Bridge Matching for Fast Image-to-Image Translation". The model is designed for fast image relighting, transforming the lighting of objects in an image. + +LBM offers: +* Fast processing with a single inference step +* High-quality relighting effects +* Memory-efficient operation +* Consistent results across various image types + +The model is trained on a diverse dataset of images with different lighting conditions, ensuring: +* Balanced representation across different image types +* High accuracy in various scenarios +* Robust performance with complex lighting + +## Roadmap + +Future plans for this repository include: +* LBM Depth - for depth map estimation +* LBM Normal - for normal map generation +* Additional optimization options + +## Requirements + +* ComfyUI +* Python 3.10+ +* Required packages (automatically installed via requirements.txt): + * torch>=2.0.0 + * torchvision>=0.15.0 + * Pillow>=9.0.0 + * numpy>=1.22.0 + * huggingface-hub>=0.19.0 + * tqdm>=4.65.0 + +## Credits + +* LBM Model: [Hugging Face Model](https://huggingface.co/jasperai/LBM_relighting) +* Original Implementation: [GitHub Repository](https://github.com/gojasper/LBM) +* Paper: "LBM: Latent Bridge Matching for Fast Image-to-Image Translation" by ClΓ©ment Chadebec, Onur Tasar, Sanjeev Sreetharan, and Benjamin Aubin +* Created by: 1038lab + +## License + +This repository's code is released under the GNU General Public License v3.0 (GPL-3.0). + +The LBM model itself is released under the Creative Commons BY-NC 4.0 license, following the original LBM implementation. Please refer to the [original repository](https://github.com/gojasper/LBM) for more details on model usage restrictions. \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..4e14c2a --- /dev/null +++ b/__init__.py @@ -0,0 +1,3 @@ +from .LBM_Relighting import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] \ No newline at end of file diff --git a/example_workflows/LBM-RElighting.json b/example_workflows/LBM-RElighting.json new file mode 100644 index 0000000..5bf155b --- /dev/null +++ b/example_workflows/LBM-RElighting.json @@ -0,0 +1,1053 @@ +{ + "id": "394ed254-7306-42a2-9ae6-aa880ce4456d", + "revision": 0, + "last_node_id": 1965, + "last_link_id": 5603, + "nodes": [ + { + "id": 1947, + "type": "MarkdownNote", + "pos": [ + 2030, + 2740 + ], + "size": [ + 540, + 170 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "widget_ue_connectable": {} + }, + "widgets_values": [ + "**ComfyUI-LBM**\n\nA ComfyUI implementation of Latent Bridge Matching (LBM) for efficient image relighting. This node utilizes the LBM algorithm to perform single-step image-to-image translation specifically for relighting tasks.\n\n[https://github.com/1038lab/ComfyUI-LBM](https://github.com/1038lab/ComfyUI-LBM)\n\nLBM Relighting Model:\n\n[https://huggingface.co/jasperai/LBM_relighting/blob/main/model.safetensors](https://huggingface.co/jasperai/LBM_relighting/blob/main/model.safetensors)" + ], + "color": "#332922", + "bgcolor": "#593930" + }, + { + "id": 1948, + "type": "RMBG", + "pos": [ + 1370, + 1800 + ], + "size": [ + 320, + 302 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 5565 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 5573 + ] + }, + { + "name": "MASK_IMAGE", + "type": "IMAGE", + "links": null + } + ], + "properties": { + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "cnr_id": "comfyui-rmbg", + "ver": "4faa7c7c12147db95345ee78b30f38273443ff99", + "Node name for S&R": "RMBG", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "RMBG-2.0", + 1, + 1024, + 0, + 0, + false, + true, + "Color", + "#222222" + ], + "color": "#222e40", + "bgcolor": "#364254" + }, + { + "id": 1949, + "type": "AILab_LoadImage", + "pos": [ + 1020, + 1800 + ], + "size": [ + 310, + 780 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 5565, + 5574 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + }, + { + "name": "MASK_IMAGE", + "type": "IMAGE", + "links": null + }, + { + "name": "WIDTH", + "type": "INT", + "links": null + }, + { + "name": "HEIGHT", + "type": "INT", + "links": null + } + ], + "properties": { + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "cnr_id": "comfyui-rmbg", + "ver": "4faa7c7c12147db95345ee78b30f38273443ff99", + "Node name for S&R": "AILab_LoadImage", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "00023-449558058.jpg", + "alpha", + 1, + "longest_side", + 1024, + "image" + ], + "color": "#2e3e57", + "bgcolor": "#4b5b73" + }, + { + "id": 1950, + "type": "AILab_LoadImage", + "pos": [ + 1020, + 2620 + ], + "size": [ + 610, + 590 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 5578 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + }, + { + "name": "MASK_IMAGE", + "type": "IMAGE", + "links": null + }, + { + "name": "WIDTH", + "type": "INT", + "links": null + }, + { + "name": "HEIGHT", + "type": "INT", + "links": null + } + ], + "properties": { + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "cnr_id": "comfyui-rmbg", + "ver": "4faa7c7c12147db95345ee78b30f38273443ff99", + "Node name for S&R": "AILab_LoadImage", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "lb2.jpg", + "alpha", + 1, + "shortest_side", + 1024, + "image" + ], + "color": "#2e3e57", + "bgcolor": "#4b5b73" + }, + { + "id": 1951, + "type": "AILab_ImageCombiner", + "pos": [ + 1810, + 2050 + ], + "size": [ + 281.7154235839844, + 194 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "foreground", + "type": "IMAGE", + "link": 5574 + }, + { + "name": "background", + "type": "IMAGE", + "link": 5578 + }, + { + "name": "foreground_mask", + "shape": 7, + "type": "MASK", + "link": 5573 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 5596, + 5603 + ] + } + ], + "properties": { + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "cnr_id": "comfyui-rmbg", + "ver": "4faa7c7c12147db95345ee78b30f38273443ff99", + "Node name for S&R": "AILab_ImageCombiner", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "normal", + 1, + 1.0000000000000002, + 50, + 50 + ], + "color": "#2e3e57", + "bgcolor": "#4b5b73" + }, + { + "id": 1962, + "type": "LBM_Relighting", + "pos": [ + 2217.97021484375, + 2530 + ], + "size": [ + 352.0298767089844, + 154 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 5596 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 5602 + ] + } + ], + "properties": { + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "Node name for S&R": "LBM_Relighting", + "widget_ue_connectable": {} + }, + "widgets_values": [ + "LBM_relighting.safetensors", + 20, + "bf16", + 0.005000000000000001, + 1 + ], + "color": "#323", + "bgcolor": "#535" + }, + { + "id": 1964, + "type": "AILab_ImagePreview", + "pos": [ + 2610, + 1800 + ], + "size": [ + 1150, + 680 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 5603 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": null + } + ], + "properties": { + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "cnr_id": "comfyui-rmbg", + "ver": "4faa7c7c12147db95345ee78b30f38273443ff99", + "Node name for S&R": "AILab_ImagePreview", + "widget_ue_connectable": {} + }, + "widgets_values": [], + "color": "#2e3e57", + "bgcolor": "#4b5b73" + }, + { + "id": 1965, + "type": "AILab_ImagePreview", + "pos": [ + 2610, + 2530 + ], + "size": [ + 1150, + 680 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 5602 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": null + } + ], + "properties": { + "enableTabs": false, + "tabWidth": 65, + "tabXOffset": 10, + "hasSecondTab": false, + "secondTabText": "Send Back", + "secondTabOffset": 80, + "secondTabWidth": 65, + "cnr_id": "comfyui-rmbg", + "ver": "4faa7c7c12147db95345ee78b30f38273443ff99", + "widget_ue_connectable": {}, + "Node name for S&R": "AILab_ImagePreview" + }, + "widgets_values": [], + "color": "#323", + "bgcolor": "#535" + } + ], + "links": [ + [ + 5565, + 1949, + 0, + 1948, + 0, + "IMAGE" + ], + [ + 5573, + 1948, + 1, + 1951, + 2, + "MASK" + ], + [ + 5574, + 1949, + 0, + 1951, + 0, + "IMAGE" + ], + [ + 5578, + 1950, + 0, + 1951, + 1, + "IMAGE" + ], + [ + 5596, + 1951, + 0, + 1962, + 0, + "IMAGE" + ], + [ + 5602, + 1962, + 0, + 1965, + 0, + "IMAGE" + ], + [ + 5603, + 1951, + 0, + 1964, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.35049389948139237, + "offset": [ + 228.81518444741147, + -1203.854919850301 + ] + }, + "frontendVersion": "1.19.9", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true, + "prompt": { + "6": { + "inputs": { + "text": "", + "clip": [ + "38", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Positive Prompt)" + } + }, + "7": { + "inputs": { + "text": "low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly", + "clip": [ + "38", + 0 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Negative Prompt)" + } + }, + "38": { + "inputs": { + "clip_name": "t5xxl_fp16.safetensors", + "type": "ltxv", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP" + } + }, + "44": { + "inputs": { + "ckpt_name": "ltx-video-13b-distilled-step-13000.safetensors" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "73": { + "inputs": { + "sampler_name": "euler_ancestral" + }, + "class_type": "KSamplerSelect", + "_meta": { + "title": "KSamplerSelect" + } + }, + "1206": { + "inputs": { + "image": "5aa.png" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "1241": { + "inputs": { + "frame_rate": 24.000000000000004, + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ] + }, + "class_type": "LTXVConditioning", + "_meta": { + "title": "LTXVConditioning" + } + }, + "1335": { + "inputs": { + "samples": [ + "1338", + 0 + ], + "vae": [ + "1870", + 0 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "1336": { + "inputs": { + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "ltxv-base", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": false, + "images": [ + "1335", + 0 + ] + }, + "class_type": "VHS_VideoCombine", + "_meta": { + "title": "Video Combine πŸŽ₯πŸ…₯πŸ…—πŸ…’" + } + }, + "1338": { + "inputs": { + "width": 768, + "height": 512, + "num_frames": 97, + "optional_cond_indices": "0, 40, 90", + "strength": 0.8, + "crop": "center", + "crf": 30, + "blur": 1, + "model": [ + "44", + 0 + ], + "vae": [ + "44", + 2 + ], + "guider": [ + "1807", + 0 + ], + "sampler": [ + "73", + 0 + ], + "sigmas": [ + "1872", + 0 + ], + "noise": [ + "1507", + 0 + ], + "optional_cond_images": [ + "1876", + 0 + ] + }, + "class_type": "LTXVBaseSampler", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ LTXV Base Sampler" + } + }, + "1507": { + "inputs": { + "noise_seed": 108 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + }, + "1593": { + "inputs": { + "factor": 0.25, + "latents": [ + "1691", + 0 + ], + "reference": [ + "1338", + 0 + ] + }, + "class_type": "LTXVAdainLatent", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ LTXV Adain Latent" + } + }, + "1598": { + "inputs": { + "noise_seed": 414 + }, + "class_type": "RandomNoise", + "_meta": { + "title": "RandomNoise" + } + }, + "1599": { + "inputs": { + "frame_rate": 24, + "loop_count": 0, + "filename_prefix": "ltxv-hd", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 18, + "save_metadata": false, + "pingpong": false, + "save_output": false, + "images": [ + "1699", + 0 + ] + }, + "class_type": "VHS_VideoCombine", + "_meta": { + "title": "Video Combine πŸŽ₯πŸ…₯πŸ…—πŸ…’" + } + }, + "1601": { + "inputs": { + "tile_size": 1280, + "overlap": 128, + "temporal_size": 128, + "temporal_overlap": 32, + "samples": [ + "1873", + 0 + ], + "vae": [ + "1870", + 0 + ] + }, + "class_type": "VAEDecodeTiled", + "_meta": { + "title": "VAE Decode (Tiled)" + } + }, + "1661": { + "inputs": { + "width": 1280, + "height": 1280, + "upscale_method": "bicubic", + "keep_proportion": true, + "divisible_by": 2, + "crop": "center", + "image": [ + "1601", + 0 + ] + }, + "class_type": "ImageResizeKJ", + "_meta": { + "title": "Resize Image" + } + }, + "1691": { + "inputs": { + "samples": [ + "1338", + 0 + ], + "upscale_model": [ + "1828", + 0 + ], + "vae": [ + "44", + 2 + ] + }, + "class_type": "LTXVLatentUpsampler", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ LTXV Latent Upsampler" + } + }, + "1699": { + "inputs": { + "grain_intensity": 0.010000000000000002, + "saturation": 0.5, + "images": [ + "1661", + 0 + ] + }, + "class_type": "LTXVFilmGrain", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ LTXV Film Grain" + } + }, + "1807": { + "inputs": { + "skip_steps_sigma_threshold": 0.9970000000000002, + "cfg_star_rescale": true, + "sigmas": "1.0, 0.9933, 0.9850, 0.9767, 0.9008, 0.6180", + "cfg_values": "1,1,1,1,1,1", + "stg_scale_values": "0,0,0,0,0,0", + "stg_rescale_values": "1, 1, 1, 1, 1, 1", + "stg_layers_indices": "[35], [35], [35], [42], [42], [42]", + "model": [ + "44", + 0 + ], + "positive": [ + "1241", + 0 + ], + "negative": [ + "1241", + 1 + ] + }, + "class_type": "STGGuiderAdvanced", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ STG Guider Advanced" + } + }, + "1813": { + "inputs": { + "skip_steps_sigma_threshold": 0.9970000000000002, + "cfg_star_rescale": true, + "sigmas": "1", + "cfg_values": "1", + "stg_scale_values": "0", + "stg_rescale_values": "1", + "stg_layers_indices": "[42]", + "model": [ + "44", + 0 + ], + "positive": [ + "1241", + 0 + ], + "negative": [ + "1241", + 1 + ] + }, + "class_type": "STGGuiderAdvanced", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ STG Guider Advanced" + } + }, + "1828": { + "inputs": { + "upscale_model": "ltxv-spatial-upscaler-0.9.7.safetensors", + "spatial_upsample": true, + "temporal_upsample": false + }, + "class_type": "LTXVLatentUpsamplerModelLoader", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ LTXV Latent Upsampler Model Loader" + } + }, + "1865": { + "inputs": { + "image": "5B.png" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "1866": { + "inputs": { + "image": "5C.png" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "1867": { + "inputs": { + "image1": [ + "1206", + 0 + ], + "image2": [ + "1865", + 0 + ] + }, + "class_type": "ImageBatch", + "_meta": { + "title": "Batch Images" + } + }, + "1868": { + "inputs": { + "image1": [ + "1867", + 0 + ], + "image2": [ + "1866", + 0 + ] + }, + "class_type": "ImageBatch", + "_meta": { + "title": "Batch Images" + } + }, + "1870": { + "inputs": { + "timestep": 0.05, + "scale": 0.025, + "seed": 42, + "vae": [ + "44", + 2 + ] + }, + "class_type": "Set VAE Decoder Noise", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ Set VAE Decoder Noise" + } + }, + "1871": { + "inputs": { + "string": "1.0000, 0.9937, 0.9875, 0.9812, 0.9750, 0.9094, 0.7250, 0.4219, 0.0" + }, + "class_type": "StringToFloatList", + "_meta": { + "title": "String to Float List" + } + }, + "1872": { + "inputs": { + "float_list": [ + "1871", + 0 + ] + }, + "class_type": "FloatToSigmas", + "_meta": { + "title": "Float To Sigmas" + } + }, + "1873": { + "inputs": { + "horizontal_tiles": 1, + "vertical_tiles": 1, + "overlap": 1, + "latents_cond_strength": 0.15, + "boost_latent_similarity": false, + "crop": "disabled", + "optional_cond_indices": "0, 40, 90", + "images_cond_strengths": "0.9", + "model": [ + "44", + 0 + ], + "vae": [ + "44", + 2 + ], + "noise": [ + "1598", + 0 + ], + "sampler": [ + "73", + 0 + ], + "sigmas": [ + "1875", + 0 + ], + "guider": [ + "1813", + 0 + ], + "latents": [ + "1593", + 0 + ], + "optional_cond_images": [ + "1876", + 0 + ] + }, + "class_type": "LTXVTiledSampler", + "_meta": { + "title": "πŸ…›πŸ…£πŸ…§ LTXV Tiled Sampler" + } + }, + "1874": { + "inputs": { + "string": "0.85, 0.7250, 0.6, 0.4219, 0.0" + }, + "class_type": "StringToFloatList", + "_meta": { + "title": "String to Float List" + } + }, + "1875": { + "inputs": { + "float_list": [ + "1874", + 0 + ] + }, + "class_type": "FloatToSigmas", + "_meta": { + "title": "Float To Sigmas" + } + }, + "1876": { + "inputs": { + "radius_x": 1, + "radius_y": 1, + "images": [ + "1868", + 0 + ] + }, + "class_type": "BlurImageFast", + "_meta": { + "title": "Blur Image (Fast)" + } + } + }, + "comfy_fork_version": "develop@580b3007", + "workspace_info": { + "id": "elBQFQknIoLYTEwIloQuw" + }, + "node_versions": { + "comfy-core": "0.3.20" + }, + "ue_links": [], + "links_added_by_ue": [] + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/LBM-RElighting.png b/example_workflows/LBM-RElighting.png new file mode 100644 index 0000000..bf4d39a Binary files /dev/null and b/example_workflows/LBM-RElighting.png differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..145194d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +diffusers>=0.19.0 +accelerate>=0.20.0 +torch>=2.0.0 +torchvision>=0.15.0 +tqdm>=4.65.0 +Pillow>=9.0.0 +transformers>=4.30.0 +safetensors>=0.3.1 +requests>=2.25.0 \ No newline at end of file