From 0f0e2bd5ab2e67a6a4707c5c9fd2219707fedabb Mon Sep 17 00:00:00 2001 From: Bubbliiiing <47347516+bubbliiiing@users.noreply.github.com> Date: Tue, 3 Feb 2026 10:23:10 +0800 Subject: [PATCH] Update Flux2 Control Cfg Distill && Fix Bug in Lora Training Register Hook (#445) --- comfyui/flux2/README.md | 2 +- comfyui/flux2/nodes.py | 4 +- ..._chunked_loading_workflow_t2i_control.json | 2 +- ...nked_loading_workflow_t2i_control_ref.json | 2 +- ..._chunked_loading_workflow_t2i_inpaint.json | 2 +- .../flux2/v1/flux2_workflow_t2i_control.json | 2 +- .../v1/flux2_workflow_t2i_control_ref.json | 2 +- .../flux2/v1/flux2_workflow_t2i_inpaint.json | 2 +- comfyui/z_image/README.md | 41 +- comfyui/z_image/nodes.py | 35 +- ..._chunked_loading_workflow_i2i_inpaint.json | 12 +- ...ked_loading_workflow_i2i_inpaint_lite.json | 12 +- .../z_image_chunked_loading_workflow_t2i.json | 10 +- ..._chunked_loading_workflow_t2i_control.json | 12 +- ...ng_workflow_t2i_control_canny_process.json | 12 +- ...ng_workflow_t2i_control_depth_process.json | 12 +- ...ked_loading_workflow_t2i_control_lite.json | 12 +- ...ing_workflow_t2i_control_pose_process.json | 12 +- ..._chunked_loading_workflow_i2i_inpaint.json | 703 ++++++ ...ked_loading_workflow_i2i_inpaint_lite.json | 704 ++++++ ...ge_turbo_chunked_loading_workflow_t2i.json | 504 +++++ ..._chunked_loading_workflow_t2i_control.json | 614 ++++++ ...ng_workflow_t2i_control_canny_process.json | 696 ++++++ ...ng_workflow_t2i_control_depth_process.json | 692 ++++++ ...ked_loading_workflow_t2i_control_lite.json | 614 ++++++ ...ing_workflow_t2i_control_pose_process.json | 692 ++++++ ..._turbo_chunked_loading_workflow_tile.json} | 0 ...o_chunked_loading_workflow_tile_lite.json} | 0 .../v1/z_image_turbo_workflow_t2i.json | 320 +++ .../z_image_turbo_workflow_t2i_control.json | 431 ++++ comfyui/z_image/v1/z_image_workflow_t2i.json | 6 +- .../v1/z_image_workflow_t2i_control.json | 8 +- examples/flux2_fun/predict_i2i_inpaint.py | 2 +- examples/flux2_fun/predict_t2i_control.py | 2 +- examples/flux2_fun/predict_t2i_control_ref.py | 2 +- examples/z_image/predict_t2i.py | 4 +- .../z_image_fun/predict_i2i_inpaint_2.1.py | 8 +- .../predict_i2i_inpaint_2.1_lite.py | 8 +- .../z_image_fun/predict_t2i_control_2.1.py | 8 +- .../predict_t2i_control_2.1_lite.py | 8 +- ....0.py => predict_turbo_i2i_inpaint_2.0.py} | 4 +- .../predict_turbo_i2i_inpaint_2.1.py | 241 +++ .../predict_turbo_i2i_inpaint_2.1_lite.py | 241 +++ ...e_2.1.py => predict_turbo_i2i_tile_2.1.py} | 4 +- ....py => predict_turbo_i2i_tile_2.1_lite.py} | 4 +- ...ontrol.py => predict_turbo_t2i_control.py} | 4 +- ....0.py => predict_turbo_t2i_control_2.0.py} | 4 +- .../predict_turbo_t2i_control_2.1.py | 241 +++ .../predict_turbo_t2i_control_2.1_lite.py | 241 +++ scripts/flux/train_lora.py | 3 - scripts/flux2/train_lora.py | 3 - scripts/flux2_fun/README_TRAIN.md | 6 +- scripts/flux2_fun/train_control.sh | 2 +- scripts/flux2_fun/train_control_distill.py | 1898 +++++++++++++++++ scripts/flux2_fun/train_control_distill.sh | 37 + scripts/hunyuanvideo/train_lora.py | 3 - scripts/longcatvideo/train_lora.py | 3 - scripts/qwenimage/train_edit_lora.py | 3 - scripts/qwenimage/train_lora.py | 3 - scripts/wan2.1/train_distill.py | 2 + scripts/wan2.1/train_distill_lora.py | 5 +- scripts/wan2.1/train_lora.py | 3 - scripts/wan2.1_fun/train_control_lora.py | 3 - scripts/wan2.1_fun/train_lora.py | 3 - scripts/wan2.2/train_animate_lora.py | 3 - scripts/wan2.2/train_distill_lora.py | 3 - scripts/wan2.2/train_lora.py | 3 - scripts/wan2.2/train_s2v_lora.py | 3 - scripts/wan2.2_fun/train_control_lora.py | 3 - scripts/wan2.2_fun/train_lora.py | 3 - scripts/z_image/train_lora.py | 3 - scripts/z_image_fun/train_control_2.1.sh | 4 +- scripts/z_image_fun/train_control_distill.py | 2 + ...rain_control.sh => train_turbo_control.sh} | 0 ...trol_2.0.sh => train_turbo_control_2.0.sh} | 0 .../z_image_fun/train_turbo_control_2.1.sh | 35 + ...till.sh => train_turbo_control_distill.sh} | 0 .../models/flux2_transformer2d_control.py | 15 +- .../models/z_image_transformer2d_control.py | 16 +- videox_fun/pipeline/pipeline_flux2.py | 28 +- videox_fun/pipeline/pipeline_flux2_control.py | 31 +- 81 files changed, 9151 insertions(+), 176 deletions(-) create mode 100644 comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint.json create mode 100644 comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint_lite.json create mode 100644 comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i.json create mode 100644 comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control.json create mode 100644 comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_canny_process.json create mode 100644 comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_depth_process.json create mode 100644 comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_lite.json create mode 100644 comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_pose_process.json rename comfyui/z_image/v1/{z_image_chunked_loading_workflow_tile.json => z_image_turbo_chunked_loading_workflow_tile.json} (100%) rename comfyui/z_image/v1/{z_image_chunked_loading_workflow_tile_lite.json => z_image_turbo_chunked_loading_workflow_tile_lite.json} (100%) create mode 100644 comfyui/z_image/v1/z_image_turbo_workflow_t2i.json create mode 100644 comfyui/z_image/v1/z_image_turbo_workflow_t2i_control.json rename examples/z_image_fun/{predict_i2i_inpaint_2.0.py => predict_turbo_i2i_inpaint_2.0.py} (99%) create mode 100644 examples/z_image_fun/predict_turbo_i2i_inpaint_2.1.py create mode 100644 examples/z_image_fun/predict_turbo_i2i_inpaint_2.1_lite.py rename examples/z_image_fun/{predict_i2i_tile_2.1.py => predict_turbo_i2i_tile_2.1.py} (99%) rename examples/z_image_fun/{predict_i2i_tile_2.1_lite.py => predict_turbo_i2i_tile_2.1_lite.py} (99%) rename examples/z_image_fun/{predict_t2i_control.py => predict_turbo_t2i_control.py} (98%) rename examples/z_image_fun/{predict_t2i_control_2.0.py => predict_turbo_t2i_control_2.0.py} (99%) create mode 100644 examples/z_image_fun/predict_turbo_t2i_control_2.1.py create mode 100644 examples/z_image_fun/predict_turbo_t2i_control_2.1_lite.py create mode 100644 scripts/flux2_fun/train_control_distill.py create mode 100644 scripts/flux2_fun/train_control_distill.sh rename scripts/z_image_fun/{train_control.sh => train_turbo_control.sh} (100%) rename scripts/z_image_fun/{train_control_2.0.sh => train_turbo_control_2.0.sh} (100%) create mode 100644 scripts/z_image_fun/train_turbo_control_2.1.sh rename scripts/z_image_fun/{train_control_distill.sh => train_turbo_control_distill.sh} (100%) diff --git a/comfyui/flux2/README.md b/comfyui/flux2/README.md index b9f5cb5..aff58cd 100644 --- a/comfyui/flux2/README.md +++ b/comfyui/flux2/README.md @@ -37,7 +37,7 @@ For chunked loading, it is recommended to directly download the FLUX.2-dev weigh │ ├── 📂 Fun_Models/ │ │ └── flux2_tokenizer/ │ └── 📂 model_patches/ -│ └── FLUX.2-dev-Fun-Controlnet-Union.safetensors +│ └── FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors ``` ### 2. Preprocessing Weights (Optional) diff --git a/comfyui/flux2/nodes.py b/comfyui/flux2/nodes.py index bdfdaa8..948f6be 100644 --- a/comfyui/flux2/nodes.py +++ b/comfyui/flux2/nodes.py @@ -875,7 +875,7 @@ class LoadFlux2ControlNetInPipeline: ), "model_name": ( folder_paths.get_filename_list("model_patches"), - {"default": "FLUX.2-dev-Fun-Controlnet-Union.safetensors", }, + {"default": "FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors", }, ), "funmodels": ("FunModels",), }, @@ -1017,7 +1017,7 @@ class LoadFlux2ControlNetInModel: ), "model_name": ( folder_paths.get_filename_list("model_patches"), - {"default": "FLUX.2-dev-Fun-Controlnet-Union.safetensors", }, + {"default": "FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors", }, ), "transformer": ("TransformerModel",), }, diff --git a/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_control.json b/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_control.json index 6855398..b0615a0 100644 --- a/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_control.json +++ b/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_control.json @@ -194,7 +194,7 @@ }, "widgets_values": [ "flux2/flux2_control.yaml", - "FLUX.2-dev-Fun-Controlnet-Union.safetensors" + "FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" ] }, { diff --git a/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_control_ref.json b/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_control_ref.json index fb06ea5..4526bc8 100644 --- a/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_control_ref.json +++ b/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_control_ref.json @@ -316,7 +316,7 @@ }, "widgets_values": [ "flux2/flux2_control.yaml", - "FLUX.2-dev-Fun-Controlnet-Union.safetensors" + "FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" ] }, { diff --git a/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_inpaint.json b/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_inpaint.json index f0caded..41808eb 100644 --- a/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_inpaint.json +++ b/comfyui/flux2/v1/flux2_chunked_loading_workflow_t2i_inpaint.json @@ -194,7 +194,7 @@ }, "widgets_values": [ "flux2/flux2_control.yaml", - "FLUX.2-dev-Fun-Controlnet-Union.safetensors" + "FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" ] }, { diff --git a/comfyui/flux2/v1/flux2_workflow_t2i_control.json b/comfyui/flux2/v1/flux2_workflow_t2i_control.json index 1250ce0..6bd1c2d 100644 --- a/comfyui/flux2/v1/flux2_workflow_t2i_control.json +++ b/comfyui/flux2/v1/flux2_workflow_t2i_control.json @@ -186,7 +186,7 @@ }, "widgets_values": [ "flux2/flux2_control.yaml", - "FLUX.2-dev-Fun-Controlnet-Union.safetensors" + "FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" ] }, { diff --git a/comfyui/flux2/v1/flux2_workflow_t2i_control_ref.json b/comfyui/flux2/v1/flux2_workflow_t2i_control_ref.json index a9df604..2051617 100644 --- a/comfyui/flux2/v1/flux2_workflow_t2i_control_ref.json +++ b/comfyui/flux2/v1/flux2_workflow_t2i_control_ref.json @@ -149,7 +149,7 @@ }, "widgets_values": [ "flux2/flux2_control.yaml", - "FLUX.2-dev-Fun-Controlnet-Union.safetensors" + "FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" ] }, { diff --git a/comfyui/flux2/v1/flux2_workflow_t2i_inpaint.json b/comfyui/flux2/v1/flux2_workflow_t2i_inpaint.json index 1d138d0..faaeff0 100644 --- a/comfyui/flux2/v1/flux2_workflow_t2i_inpaint.json +++ b/comfyui/flux2/v1/flux2_workflow_t2i_inpaint.json @@ -122,7 +122,7 @@ }, "widgets_values": [ "flux2/flux2_control.yaml", - "FLUX.2-dev-Fun-Controlnet-Union.safetensors" + "FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" ] }, { diff --git a/comfyui/z_image/README.md b/comfyui/z_image/README.md index 5efe992..bc7db53 100644 --- a/comfyui/z_image/README.md +++ b/comfyui/z_image/README.md @@ -1,4 +1,4 @@ -# Z-Image-Turbo Model Setup Guide +# Z-Image Model Setup Guide ## a. Model Links and Storage Locations @@ -13,7 +13,7 @@ For chunked loading, it is recommended to directly download the Z-Image weights | Component | File Name | |-----------|-----------| | Text Encoder | [`qwen_3_4b.safetensors`](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/text_encoders/qwen_3_4b.safetensors) | -| Diffusion Model | [`z_image_turbo_bf16.safetensors`](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/diffusion_models/z_image_turbo_bf16.safetensors) | +| Diffusion Model | [`z_image_turbo_bf16.safetensors`](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/diffusion_models/z_image_turbo_bf16.safetensors) and [`z_image_bf16.safetensors`](https://huggingface.co/Comfy-Org/z_image/resolve/main/split_files/diffusion_models/z_image_bf16.safetensors) | | VAE | [`ae.safetensors`](https://huggingface.co/Comfy-Org/z_image_turbo/resolve/main/split_files/vae/ae.safetensors) | | tokenizer(Qwen3-4B) | [`tokenizer`](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo/tree/main/tokenizer) | @@ -23,6 +23,7 @@ For chunked loading, it is recommended to directly download the Z-Image weights |------|--------------|-------------|-------------| | Z-Image-Turbo-Fun-Controlnet-Union | [🤗Link](https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union) | [😄Link](https://modelscope.cn/models/PAI/Z-Image-Turbo-Fun-Controlnet-Union) | ControlNet weights for Z-Image-Turbo, supporting multiple control conditions including Canny, Depth, Pose, MLSD, etc. | | Z-Image-Turbo-Fun-Controlnet-Union-2.1 | [🤗Link](https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union-2.1) | [😄Link](https://modelscope.cn/models/PAI/Z-Image-Turbo-Fun-Controlnet-Union-2.1) | Upgraded ControlNet weights for Z-Image-Turbo with additions at more layers and longer training time, supporting multiple control conditions including Canny, Depth, Pose, MLSD, etc. | +| Z-Image-Fun-Controlnet-Union-2.1 | [🤗Link](https://huggingface.co/alibaba-pai/Z-Image-Fun-Controlnet-Union-2.1) | [😄Link](https://modelscope.cn/models/PAI/Z-Image-Fun-Controlnet-Union-2.1) | Upgraded ControlNet weights for Z-Image with additions at more layers and longer training time, supporting multiple control conditions including Canny, Depth, Pose, MLSD, Scribble, Hed and Gray. | **Storage Location:** @@ -74,8 +75,9 @@ If you prefer full model loading, you can directly download the diffusers weight | Name | Hugging Face | Model Scope | Description | |------|--------------|-------------|-------------| | Z-Image-Turbo | [🤗Link](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo) | [😄Link](https://www.modelscope.cn/models/Tongyi-MAI/Z-Image-Turbo) | Official full weights for Z-Image-Turbo | +| Z-Image | [🤗Link](https://huggingface.co/Tongyi-MAI/Z-Image) | [😄Link](https://www.modelscope.cn/models/Tongyi-MAI/Z-Image) | Official full weights for Z-Image | -For full model loading, use the diffusers version of Z-Image Turbo and place the model in `ComfyUI/models/Fun_Models/`. +For full model loading, use the diffusers version of Z-Image and place the model in `ComfyUI/models/Fun_Models/`. **Storage Location:** @@ -83,6 +85,7 @@ For full model loading, use the diffusers version of Z-Image Turbo and place the 📂 ComfyUI/ ├── 📂 models/ │ └── 📂 Fun_Models/ +│ ├── 📂 Z-Image │ └── 📂 Z-Image-Turbo ``` @@ -90,20 +93,36 @@ For full model loading, use the diffusers version of Z-Image Turbo and place the ### 1. Chunked Loading (Recommended) -[Z Image Turbo Text to Image](v1/z_image_chunked_loading_workflow_t2i.json) +[Z Image Text to Image](v1/z_image_chunked_loading_workflow_t2i.json) -[Z Image Turbo Text to Image and Control](v1/z_image_chunked_loading_workflow_t2i_control.json) +[Z Image Text to Image and Control](v1/z_image_chunked_loading_workflow_t2i_control.json) -[Z Image Turbo Text to Image and Control with Pose Detect](v1/z_image_chunked_loading_workflow_t2i_control_pose_process.json) +[Z Image Text to Image and Control with Pose Detect](v1/z_image_chunked_loading_workflow_t2i_control_pose_process.json) -[Z Image Turbo Text to Image and Control with Depth Detect](v1/z_image_chunked_loading_workflow_t2i_control_depth_process.json) +[Z Image Text to Image and Control with Depth Detect](v1/z_image_chunked_loading_workflow_t2i_control_depth_process.json) -[Z Image Turbo Text to Image and Control with Canny Detect](v1/z_image_chunked_loading_workflow_t2i_control_canny_process.json) +[Z Image Text to Image and Control with Canny Detect](v1/z_image_chunked_loading_workflow_t2i_control_canny_process.json) -[Z Image Turbo Image to Image with Inpaint](v1/z_image_chunked_loading_workflow_i2i_inpaint.json) +[Z Image Image to Image with Inpaint](v1/z_image_chunked_loading_workflow_i2i_inpaint.json) + +[Z Image Turbo Text to Image](v1/z_image_turbo_chunked_loading_workflow_t2i.json) + +[Z Image Turbo Text to Image and Control](v1/z_image_turbo_chunked_loading_workflow_t2i_control.json) + +[Z Image Turbo Text to Image and Control with Pose Detect](v1/z_image_turbo_chunked_loading_workflow_t2i_control_pose_process.json) + +[Z Image Turbo Text to Image and Control with Depth Detect](v1/z_image_turbo_chunked_loading_workflow_t2i_control_depth_process.json) + +[Z Image Turbo Text to Image and Control with Canny Detect](v1/z_image_turbo_chunked_loading_workflow_t2i_control_canny_process.json) + +[Z Image Turbo Image to Image with Inpaint](v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint.json) ### 2. Full Model Loading (Optional) -[Z Image Turbo Text to Image](v1/z_image_workflow_t2i.json) +[Z Image Text to Image](v1/z_image_workflow_t2i.json) -[Z Image Turbo Text to Image and Control](v1/z_image_workflow_t2i_control.json) +[Z Image Text to Image and Control](v1/z_image_workflow_t2i_control.json) + +[Z Image Turbo Text to Image](v1/z_image_turbo_workflow_t2i.json) + +[Z Image Turbo Text to Image and Control](v1/z_image_turbo_workflow_t2i_control.json) diff --git a/comfyui/z_image/nodes.py b/comfyui/z_image/nodes.py index 9286453..fb5651d 100644 --- a/comfyui/z_image/nodes.py +++ b/comfyui/z_image/nodes.py @@ -27,6 +27,9 @@ from ...videox_fun.models import (AutoencoderKL, AutoTokenizer, ZImageTransformer2DModel) from ...videox_fun.models.cache_utils import get_teacache_coefficients from ...videox_fun.pipeline import ZImageControlPipeline, ZImagePipeline +from ...videox_fun.utils import (register_auto_device_hook, + safe_enable_group_offload, + safe_remove_group_offloading) from ...videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler from ...videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler from ...videox_fun.utils.fp8_optimization import ( @@ -453,7 +456,9 @@ class CombineZImagePipeline: "tokenizer": ("Tokenizer",), "model_name": ("STRING",), "GPU_memory_mode":( - ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + [ + "model_full_load", "model_full_load_and_qfloat8", "model_cpu_offload", + "model_cpu_offload_and_qfloat8", "model_group_offload", "sequential_cpu_offload"], { "default": "model_cpu_offload", } @@ -505,14 +510,17 @@ class CombineZImagePipeline: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device=offload_device, offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: @@ -536,14 +544,17 @@ class LoadZImageModel: "required": { "model": ( [ - "Z-Image-Turbo" + "Z-Image-Turbo", + "Z-Image" ], { "default": 'Z-Image-Turbo', } ), "GPU_memory_mode":( - ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + [ + "model_full_load", "model_full_load_and_qfloat8", "model_cpu_offload", + "model_cpu_offload_and_qfloat8", "model_group_offload", "sequential_cpu_offload"], { "default": "model_cpu_offload", } @@ -637,14 +648,17 @@ class LoadZImageModel: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device=offload_device, offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: @@ -800,14 +814,17 @@ class LoadZImageControlNetInPipeline: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_group_offload": + register_auto_device_hook(pipeline.transformer) + safe_enable_group_offload(pipeline, onload_device=device, offload_device=offload_device, offload_type="leaf_level", use_stream=True) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(control_transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(control_transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(control_transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(control_transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(control_transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(control_transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_i2i_inpaint.json b/comfyui/z_image/v1/z_image_chunked_loading_workflow_i2i_inpaint.json index dc93712..47477e1 100644 --- a/comfyui/z_image/v1/z_image_chunked_loading_workflow_i2i_inpaint.json +++ b/comfyui/z_image/v1/z_image_chunked_loading_workflow_i2i_inpaint.json @@ -131,7 +131,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -223,7 +223,7 @@ }, "widgets_values": [ "", - "model_cpu_offload" + "model_group_offload" ] }, { @@ -261,7 +261,7 @@ "Node name for S&R": "LoadZImageTransformerModel" }, "widgets_values": [ - "z_image_turbo_bf16.safetensors", + "z_image_bf16.safetensors", "bf16" ] }, @@ -300,7 +300,7 @@ }, "widgets_values": [ "z_image/z_image_control_2.1.yaml", - "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + "Z-Image-Fun-Controlnet-Union-2.1.safetensors" ] }, { @@ -369,8 +369,8 @@ 1568, 43, "fixed", - 8, - 0, + 25, + 4.5, "Flow", 3, 0.8 diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_i2i_inpaint_lite.json b/comfyui/z_image/v1/z_image_chunked_loading_workflow_i2i_inpaint_lite.json index af0b26c..7c56ed7 100644 --- a/comfyui/z_image/v1/z_image_chunked_loading_workflow_i2i_inpaint_lite.json +++ b/comfyui/z_image/v1/z_image_chunked_loading_workflow_i2i_inpaint_lite.json @@ -131,7 +131,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -223,7 +223,7 @@ }, "widgets_values": [ "", - "model_cpu_offload" + "model_group_offload" ] }, { @@ -261,7 +261,7 @@ "Node name for S&R": "LoadZImageTransformerModel" }, "widgets_values": [ - "z_image_turbo_bf16.safetensors", + "z_image_bf16.safetensors", "bf16" ] }, @@ -331,8 +331,8 @@ 1568, 43, "fixed", - 8, - 0, + 25, + 4.5, "Flow", 3, 0.8 @@ -535,7 +535,7 @@ }, "widgets_values": [ "z_image/z_image_control_2.1_lite.yaml", - "Z-Image-Turbo-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" + "Z-Image-Fun-Controlnet-Union-2.1-lite.safetensors" ] } ], diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i.json b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i.json index 443f25a..01983f4 100644 --- a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i.json +++ b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i.json @@ -104,8 +104,8 @@ 1568, 43, "fixed", - 8, - 0, + 25, + 4.5, "Flow", 3 ] @@ -145,7 +145,7 @@ "Node name for S&R": "LoadZImageTransformerModel" }, "widgets_values": [ - "z_image_turbo_bf16.safetensors", + "z_image_bf16.safetensors", "bf16" ] }, @@ -245,7 +245,7 @@ }, "widgets_values": [ "", - "model_cpu_offload" + "model_group_offload" ] }, { @@ -350,7 +350,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control.json b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control.json index 1e0ce79..8ec08b0 100644 --- a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control.json +++ b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control.json @@ -131,7 +131,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -255,7 +255,7 @@ }, "widgets_values": [ "", - "model_cpu_offload" + "model_group_offload" ] }, { @@ -357,7 +357,7 @@ "Node name for S&R": "LoadZImageTransformerModel" }, "widgets_values": [ - "z_image_turbo_bf16.safetensors", + "z_image_bf16.safetensors", "bf16" ] }, @@ -396,7 +396,7 @@ }, "widgets_values": [ "z_image/z_image_control_2.1.yaml", - "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + "Z-Image-Fun-Controlnet-Union-2.1.safetensors" ] }, { @@ -465,8 +465,8 @@ 1568, 43, "fixed", - 8, - 0, + 25, + 4.5, "Flow", 3, 0.8 diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_canny_process.json b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_canny_process.json index dfaf129..f2bf4e5 100644 --- a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_canny_process.json +++ b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_canny_process.json @@ -131,7 +131,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -223,7 +223,7 @@ }, "widgets_values": [ "", - "model_cpu_offload" + "model_group_offload" ] }, { @@ -261,7 +261,7 @@ "Node name for S&R": "LoadZImageTransformerModel" }, "widgets_values": [ - "z_image_turbo_bf16.safetensors", + "z_image_bf16.safetensors", "bf16" ] }, @@ -300,7 +300,7 @@ }, "widgets_values": [ "z_image/z_image_control_2.1.yaml", - "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + "Z-Image-Fun-Controlnet-Union-2.1.safetensors" ] }, { @@ -369,8 +369,8 @@ 1568, 43, "fixed", - 8, - 0, + 25, + 4.5, "Flow", 3, 0.8 diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_depth_process.json b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_depth_process.json index 3df4df9..770cfd9 100644 --- a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_depth_process.json +++ b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_depth_process.json @@ -131,7 +131,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -223,7 +223,7 @@ }, "widgets_values": [ "", - "model_cpu_offload" + "model_group_offload" ] }, { @@ -261,7 +261,7 @@ "Node name for S&R": "LoadZImageTransformerModel" }, "widgets_values": [ - "z_image_turbo_bf16.safetensors", + "z_image_bf16.safetensors", "bf16" ] }, @@ -300,7 +300,7 @@ }, "widgets_values": [ "z_image/z_image_control_2.1.yaml", - "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + "Z-Image-Fun-Controlnet-Union-2.1.safetensors" ] }, { @@ -369,8 +369,8 @@ 1568, 43, "fixed", - 8, - 0, + 25, + 4.5, "Flow", 3, 0.8 diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_lite.json b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_lite.json index 5052443..2209906 100644 --- a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_lite.json +++ b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_lite.json @@ -131,7 +131,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -290,7 +290,7 @@ "Node name for S&R": "LoadZImageTransformerModel" }, "widgets_values": [ - "z_image_turbo_bf16.safetensors", + "z_image_bf16.safetensors", "bf16" ] }, @@ -360,8 +360,8 @@ 1568, 43, "fixed", - 8, - 0, + 25, + 4.5, "Flow", 3, 0.8 @@ -431,7 +431,7 @@ }, "widgets_values": [ "", - "model_cpu_offload" + "model_group_offload" ] }, { @@ -469,7 +469,7 @@ }, "widgets_values": [ "z_image/z_image_control_2.1_lite.yaml", - "Z-Image-Turbo-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" + "Z-Image-Fun-Controlnet-Union-2.1-lite.safetensors" ] } ], diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_pose_process.json b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_pose_process.json index 2893dd2..5fecc93 100644 --- a/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_pose_process.json +++ b/comfyui/z_image/v1/z_image_chunked_loading_workflow_t2i_control_pose_process.json @@ -131,7 +131,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -223,7 +223,7 @@ }, "widgets_values": [ "", - "model_cpu_offload" + "model_group_offload" ] }, { @@ -261,7 +261,7 @@ "Node name for S&R": "LoadZImageTransformerModel" }, "widgets_values": [ - "z_image_turbo_bf16.safetensors", + "z_image_bf16.safetensors", "bf16" ] }, @@ -300,7 +300,7 @@ }, "widgets_values": [ "z_image/z_image_control_2.1.yaml", - "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + "Z-Image-Fun-Controlnet-Union-2.1.safetensors" ] }, { @@ -369,8 +369,8 @@ 1568, 43, "fixed", - 8, - 0, + 25, + 4.5, "Flow", 3, 0.8 diff --git a/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint.json new file mode 100644 index 0000000..dc93712 --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint.json @@ -0,0 +1,703 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 107, + "last_link_id": 113, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 91, + "type": "LoadZImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTextEncoderModel" + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 88 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 97, + "type": "Note", + "pos": [ + -354.4680507215508, + -433.6570714778354 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 96, + "type": "CombineZImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 96 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "CombineZImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload" + ] + }, + { + "id": 92, + "type": "LoadZImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 95 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTransformerModel" + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "bf16" + ] + }, + { + "id": 102, + "type": "LoadZImageControlNetInModel", + "pos": [ + 779.793189390101, + -457.3825558553134 + ], + "size": [ + 589.8698159570357, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 95 + } + ], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 96 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageControlNetInModel" + }, + "widgets_values": [ + "z_image/z_image_control_2.1.yaml", + "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + ] + }, + { + "id": 99, + "type": "ZImageControlSampler", + "pos": [ + 727.2482831521481, + -52.41010988674983 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 87 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 88 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "control_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "inpaint_image", + "shape": 7, + "type": "IMAGE", + "link": 110 + }, + { + "name": "mask_image", + "shape": 7, + "type": "IMAGE", + "link": 112 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageControlSampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3, + 0.8 + ] + }, + { + "id": 93, + "type": "LoadZImageVAEModel", + "pos": [ + 1168.1599508804559, + -314.8650476692169 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageVAEModel" + }, + "widgets_values": [ + "ae.safetensors", + "bf16" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1049.1402001998824, + -52.699751832945104 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 90 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 104, + "type": "PreviewImage", + "pos": [ + 911.4377073728218, + 418.9988584275326 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 113 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 107, + "type": "MaskToImage", + "pos": [ + 658.3044275669289, + 511.8717365608471 + ], + "size": [ + 140, + 26 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 111 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 112, + 113 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage" + } + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 323.5443519405259, + 489.8702544908114 + ], + "size": [ + 270, + 314.00000000000006 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 110 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 111 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImage", + "image": "clipspace/clipspace-painted-masked-1766731857414.png [input]" + }, + "widgets_values": [ + "clipspace/clipspace-painted-masked-1766731857414.png [input]", + "image" + ] + } + ], + "links": [ + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 87, + 96, + 0, + 99, + 0, + "FunModels" + ], + [ + 88, + 75, + 0, + 99, + 1, + "STRING_PROMPT" + ], + [ + 89, + 73, + 0, + 99, + 2, + "STRING_PROMPT" + ], + [ + 90, + 99, + 0, + 88, + 0, + "IMAGE" + ], + [ + 95, + 92, + 0, + 102, + 0, + "TransformerModel" + ], + [ + 96, + 102, + 0, + 96, + 0, + "TransformerModel" + ], + [ + 110, + 100, + 0, + 99, + 4, + "IMAGE" + ], + [ + 111, + 100, + 1, + 107, + 0, + "MASK" + ], + [ + 112, + 107, + 0, + 99, + 5, + "IMAGE" + ], + [ + 113, + 107, + 0, + 104, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 1350.4793699732413, + 404.87677206390265 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7772383863288174, + "offset": [ + -164.75871381690226, + 267.25116947501067 + ] + }, + "frontendVersion": "1.36.11", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "93aa7b2530dccd1e91c625eee439a5e24f8ffa04", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint_lite.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint_lite.json new file mode 100644 index 0000000..af0b26c --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_i2i_inpaint_lite.json @@ -0,0 +1,704 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 107, + "last_link_id": 113, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 91, + "type": "LoadZImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTextEncoderModel" + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 88 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 97, + "type": "Note", + "pos": [ + -354.4680507215508, + -433.6570714778354 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 96, + "type": "CombineZImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 96 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "CombineZImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload" + ] + }, + { + "id": 92, + "type": "LoadZImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 95 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTransformerModel" + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "bf16" + ] + }, + { + "id": 99, + "type": "ZImageControlSampler", + "pos": [ + 727.2482831521481, + -52.41010988674983 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 87 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 88 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "control_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "inpaint_image", + "shape": 7, + "type": "IMAGE", + "link": 110 + }, + { + "name": "mask_image", + "shape": 7, + "type": "IMAGE", + "link": 112 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageControlSampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3, + 0.8 + ] + }, + { + "id": 93, + "type": "LoadZImageVAEModel", + "pos": [ + 1168.1599508804559, + -314.8650476692169 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageVAEModel" + }, + "widgets_values": [ + "ae.safetensors", + "bf16" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1049.1402001998824, + -52.699751832945104 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 90 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 104, + "type": "PreviewImage", + "pos": [ + 911.4377073728218, + 418.9988584275326 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 113 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 107, + "type": "MaskToImage", + "pos": [ + 658.3044275669289, + 511.8717365608471 + ], + "size": [ + 140, + 26 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 111 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 112, + 113 + ] + } + ], + "properties": { + "Node name for S&R": "MaskToImage" + }, + "widgets_values": [] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 323.5443519405259, + 489.8702544908114 + ], + "size": [ + 270, + 314.00000000000006 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 110 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 111 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImage", + "image": "clipspace/clipspace-painted-masked-1766731857414.png [input]" + }, + "widgets_values": [ + "clipspace/clipspace-painted-masked-1766731857414.png [input]", + "image" + ] + }, + { + "id": 102, + "type": "LoadZImageControlNetInModel", + "pos": [ + 779.793189390101, + -457.3825558553134 + ], + "size": [ + 589.8698159570357, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 95 + } + ], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 96 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageControlNetInModel" + }, + "widgets_values": [ + "z_image/z_image_control_2.1_lite.yaml", + "Z-Image-Turbo-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" + ] + } + ], + "links": [ + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 87, + 96, + 0, + 99, + 0, + "FunModels" + ], + [ + 88, + 75, + 0, + 99, + 1, + "STRING_PROMPT" + ], + [ + 89, + 73, + 0, + 99, + 2, + "STRING_PROMPT" + ], + [ + 90, + 99, + 0, + 88, + 0, + "IMAGE" + ], + [ + 95, + 92, + 0, + 102, + 0, + "TransformerModel" + ], + [ + 96, + 102, + 0, + 96, + 0, + "TransformerModel" + ], + [ + 110, + 100, + 0, + 99, + 4, + "IMAGE" + ], + [ + 111, + 100, + 1, + 107, + 0, + "MASK" + ], + [ + 112, + 107, + 0, + 99, + 5, + "IMAGE" + ], + [ + 113, + 107, + 0, + 104, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 1350.4793699732413, + 404.87677206390265 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7772383863288174, + "offset": [ + 248.90036994067935, + 733.942567031054 + ] + }, + "frontendVersion": "1.34.9", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "07dd34b942f866d5f95e8b812b6082d359079260", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i.json new file mode 100644 index 0000000..443f25a --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i.json @@ -0,0 +1,504 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 97, + "last_link_id": 83, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1070.207763671875, + -73.63389587402344 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 77 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 95, + "type": "ZImageT2ISampler", + "pos": [ + 719.3201904296875, + -72.24609375 + ], + "size": [ + 280.724609375, + 386 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 83 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 75 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 76 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageT2ISampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3 + ] + }, + { + "id": 92, + "type": "LoadZImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 78 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTransformerModel" + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "bf16" + ] + }, + { + "id": 93, + "type": "LoadZImageVAEModel", + "pos": [ + 775.0554809570312, + -470.7688293457031 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageVAEModel" + }, + "widgets_values": [ + "ae.safetensors", + "bf16" + ] + }, + { + "id": 96, + "type": "CombineZImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 78 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 83 + ] + } + ], + "properties": { + "Node name for S&R": "CombineZImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload" + ] + }, + { + "id": 91, + "type": "LoadZImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTextEncoderModel" + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 75 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 97, + "type": "Note", + "pos": [ + -354.4680507215508, + -433.6570714778354 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 75, + 75, + 0, + 95, + 1, + "STRING_PROMPT" + ], + [ + 76, + 73, + 0, + 95, + 2, + "STRING_PROMPT" + ], + [ + 77, + 95, + 0, + 88, + 0, + "IMAGE" + ], + [ + 78, + 92, + 0, + 96, + 0, + "TransformerModel" + ], + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 83, + 96, + 0, + 95, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 985.5581665039062, + 393.7902526855469 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015777, + "offset": [ + 598.8157343153007, + 736.7507277278353 + ] + }, + "frontendVersion": "1.36.11", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "comfy-core": "0.6.0", + "CogVideoX-Fun": "244f11053106af1a58ac25fd0bbb508fd7b89c0f" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control.json new file mode 100644 index 0000000..1e0ce79 --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control.json @@ -0,0 +1,614 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 102, + "last_link_id": 96, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 91, + "type": "LoadZImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTextEncoderModel" + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 88 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 97, + "type": "Note", + "pos": [ + -354.4680507215508, + -433.6570714778354 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 93, + "type": "LoadZImageVAEModel", + "pos": [ + 1168.1599508804559, + -314.8650476692169 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageVAEModel" + }, + "widgets_values": [ + "ae.safetensors", + "bf16" + ] + }, + { + "id": 96, + "type": "CombineZImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 96 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "CombineZImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1049.1402001998824, + -52.699751832945104 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 90 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 396.48551767952716, + 419.158511044569 + ], + "size": [ + 270, + 314.00000000000006 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 86 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "a7kXeQ5l9Dhspes7q3x3G (1).png", + "image" + ] + }, + { + "id": 92, + "type": "LoadZImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 95 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTransformerModel" + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "bf16" + ] + }, + { + "id": 102, + "type": "LoadZImageControlNetInModel", + "pos": [ + 779.793189390101, + -457.3825558553134 + ], + "size": [ + 589.8698159570357, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 95 + } + ], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 96 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageControlNetInModel" + }, + "widgets_values": [ + "z_image/z_image_control_2.1.yaml", + "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + ] + }, + { + "id": 99, + "type": "ZImageControlSampler", + "pos": [ + 727.2482831521481, + -52.41010988674983 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 87 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 88 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "control_image", + "shape": 7, + "type": "IMAGE", + "link": 86 + }, + { + "name": "inpaint_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "mask_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageControlSampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3, + 0.8 + ] + } + ], + "links": [ + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 86, + 100, + 0, + 99, + 3, + "IMAGE" + ], + [ + 87, + 96, + 0, + 99, + 0, + "FunModels" + ], + [ + 88, + 75, + 0, + 99, + 1, + "STRING_PROMPT" + ], + [ + 89, + 73, + 0, + 99, + 2, + "STRING_PROMPT" + ], + [ + 90, + 99, + 0, + 88, + 0, + "IMAGE" + ], + [ + 95, + 92, + 0, + 102, + 0, + "TransformerModel" + ], + [ + 96, + 102, + 0, + 96, + 0, + "TransformerModel" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 1350.4793699732413, + 404.87677206390265 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9594420649225085, + "offset": [ + 96.69529077203181, + 609.5309015132086 + ] + }, + "frontendVersion": "1.36.11", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "244f11053106af1a58ac25fd0bbb508fd7b89c0f", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_canny_process.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_canny_process.json new file mode 100644 index 0000000..dfaf129 --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_canny_process.json @@ -0,0 +1,696 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 106, + "last_link_id": 109, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 91, + "type": "LoadZImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTextEncoderModel" + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 88 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 97, + "type": "Note", + "pos": [ + -354.4680507215508, + -433.6570714778354 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 96, + "type": "CombineZImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 96 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "CombineZImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload" + ] + }, + { + "id": 92, + "type": "LoadZImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 95 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTransformerModel" + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "bf16" + ] + }, + { + "id": 102, + "type": "LoadZImageControlNetInModel", + "pos": [ + 779.793189390101, + -457.3825558553134 + ], + "size": [ + 589.8698159570357, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 95 + } + ], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 96 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageControlNetInModel" + }, + "widgets_values": [ + "z_image/z_image_control_2.1.yaml", + "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + ] + }, + { + "id": 99, + "type": "ZImageControlSampler", + "pos": [ + 727.2482831521481, + -52.41010988674983 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 87 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 88 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "control_image", + "shape": 7, + "type": "IMAGE", + "link": 109 + }, + { + "name": "inpaint_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "mask_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageControlSampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3, + 0.8 + ] + }, + { + "id": 93, + "type": "LoadZImageVAEModel", + "pos": [ + 1168.1599508804559, + -314.8650476692169 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageVAEModel" + }, + "widgets_values": [ + "ae.safetensors", + "bf16" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1049.1402001998824, + -52.699751832945104 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 90 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 281.12436800744575, + 417.89046761218935 + ], + "size": [ + 270, + 314.00000000000006 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 107 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "z-image-turbo_00004_.png", + "image" + ] + }, + { + "id": 104, + "type": "PreviewImage", + "pos": [ + 911.4377073728218, + 418.9988584275326 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 108 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 106, + "type": "ImageToCanny", + "pos": [ + 600.6679574832448, + 416.8636458672411 + ], + "size": [ + 270, + 82 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "input_image", + "type": "IMAGE", + "link": 107 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 108, + 109 + ] + } + ], + "properties": { + "Node name for S&R": "ImageToCanny" + }, + "widgets_values": [ + 100, + 200 + ] + } + ], + "links": [ + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 87, + 96, + 0, + 99, + 0, + "FunModels" + ], + [ + 88, + 75, + 0, + 99, + 1, + "STRING_PROMPT" + ], + [ + 89, + 73, + 0, + 99, + 2, + "STRING_PROMPT" + ], + [ + 90, + 99, + 0, + 88, + 0, + "IMAGE" + ], + [ + 95, + 92, + 0, + 102, + 0, + "TransformerModel" + ], + [ + 96, + 102, + 0, + 96, + 0, + "TransformerModel" + ], + [ + 107, + 100, + 0, + 106, + 0, + "IMAGE" + ], + [ + 108, + 106, + 0, + 104, + 0, + "IMAGE" + ], + [ + 109, + 106, + 0, + 99, + 3, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 1350.4793699732413, + 404.87677206390265 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.5785382099684587, + "offset": [ + 793.7261950497102, + 754.1754224231845 + ] + }, + "frontendVersion": "1.36.11", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "93aa7b2530dccd1e91c625eee439a5e24f8ffa04", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_depth_process.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_depth_process.json new file mode 100644 index 0000000..3df4df9 --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_depth_process.json @@ -0,0 +1,692 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 105, + "last_link_id": 104, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 91, + "type": "LoadZImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTextEncoderModel" + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 88 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 97, + "type": "Note", + "pos": [ + -354.4680507215508, + -433.6570714778354 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 96, + "type": "CombineZImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 96 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "CombineZImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload" + ] + }, + { + "id": 92, + "type": "LoadZImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 95 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTransformerModel" + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "bf16" + ] + }, + { + "id": 102, + "type": "LoadZImageControlNetInModel", + "pos": [ + 779.793189390101, + -457.3825558553134 + ], + "size": [ + 589.8698159570357, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 95 + } + ], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 96 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageControlNetInModel" + }, + "widgets_values": [ + "z_image/z_image_control_2.1.yaml", + "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + ] + }, + { + "id": 99, + "type": "ZImageControlSampler", + "pos": [ + 727.2482831521481, + -52.41010988674983 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 87 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 88 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "control_image", + "shape": 7, + "type": "IMAGE", + "link": 104 + }, + { + "name": "inpaint_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "mask_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageControlSampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3, + 0.8 + ] + }, + { + "id": 93, + "type": "LoadZImageVAEModel", + "pos": [ + 1168.1599508804559, + -314.8650476692169 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageVAEModel" + }, + "widgets_values": [ + "ae.safetensors", + "bf16" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1049.1402001998824, + -52.699751832945104 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 90 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 104, + "type": "PreviewImage", + "pos": [ + 844.9628845788926, + 422.400403774606 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 103 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 281.12436800744575, + 417.89046761218935 + ], + "size": [ + 270, + 314.00000000000006 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 102 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "z-image-turbo_00004_.png", + "image" + ] + }, + { + "id": 105, + "type": "ImageToDepth", + "pos": [ + 611.889314416795, + 425.1418418025041 + ], + "size": [ + 164.7039856092299, + 28.074046747697935 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "input_image", + "type": "IMAGE", + "link": 102 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 103, + 104 + ] + } + ], + "properties": { + "Node name for S&R": "ImageToDepth" + } + } + ], + "links": [ + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 87, + 96, + 0, + 99, + 0, + "FunModels" + ], + [ + 88, + 75, + 0, + 99, + 1, + "STRING_PROMPT" + ], + [ + 89, + 73, + 0, + 99, + 2, + "STRING_PROMPT" + ], + [ + 90, + 99, + 0, + 88, + 0, + "IMAGE" + ], + [ + 95, + 92, + 0, + 102, + 0, + "TransformerModel" + ], + [ + 96, + 102, + 0, + 96, + 0, + "TransformerModel" + ], + [ + 102, + 100, + 0, + 105, + 0, + "IMAGE" + ], + [ + 103, + 105, + 0, + 104, + 0, + "IMAGE" + ], + [ + 104, + 105, + 0, + 99, + 3, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 1350.4793699732413, + 404.87677206390265 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7154376913110498, + "offset": [ + 365.0760840892105, + 597.0698647794829 + ] + }, + "frontendVersion": "1.36.11", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "93aa7b2530dccd1e91c625eee439a5e24f8ffa04", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_lite.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_lite.json new file mode 100644 index 0000000..5052443 --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_lite.json @@ -0,0 +1,614 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 102, + "last_link_id": 96, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 91, + "type": "LoadZImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTextEncoderModel" + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 88 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 97, + "type": "Note", + "pos": [ + -354.4680507215508, + -433.6570714778354 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 93, + "type": "LoadZImageVAEModel", + "pos": [ + 1168.1599508804559, + -314.8650476692169 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageVAEModel" + }, + "widgets_values": [ + "ae.safetensors", + "bf16" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1049.1402001998824, + -52.699751832945104 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 90 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 396.48551767952716, + 419.158511044569 + ], + "size": [ + 270, + 314.00000000000006 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 86 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "a7kXeQ5l9Dhspes7q3x3G (1).png", + "image" + ] + }, + { + "id": 92, + "type": "LoadZImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 95 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTransformerModel" + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "bf16" + ] + }, + { + "id": 99, + "type": "ZImageControlSampler", + "pos": [ + 727.2482831521481, + -52.41010988674983 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 87 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 88 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "control_image", + "shape": 7, + "type": "IMAGE", + "link": 86 + }, + { + "name": "inpaint_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "mask_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageControlSampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3, + 0.8 + ] + }, + { + "id": 96, + "type": "CombineZImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 96 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "CombineZImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload" + ] + }, + { + "id": 102, + "type": "LoadZImageControlNetInModel", + "pos": [ + 779.793189390101, + -457.3825558553134 + ], + "size": [ + 589.8698159570357, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 95 + } + ], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 96 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageControlNetInModel" + }, + "widgets_values": [ + "z_image/z_image_control_2.1_lite.yaml", + "Z-Image-Turbo-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" + ] + } + ], + "links": [ + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 86, + 100, + 0, + 99, + 3, + "IMAGE" + ], + [ + 87, + 96, + 0, + 99, + 0, + "FunModels" + ], + [ + 88, + 75, + 0, + 99, + 1, + "STRING_PROMPT" + ], + [ + 89, + 73, + 0, + 99, + 2, + "STRING_PROMPT" + ], + [ + 90, + 99, + 0, + 88, + 0, + "IMAGE" + ], + [ + 95, + 92, + 0, + 102, + 0, + "TransformerModel" + ], + [ + 96, + 102, + 0, + 96, + 0, + "TransformerModel" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 1350.4793699732413, + 404.87677206390265 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7208430239838531, + "offset": [ + 460.4224504078358, + 644.7360602102879 + ] + }, + "frontendVersion": "1.34.9", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "07dd34b942f866d5f95e8b812b6082d359079260", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_pose_process.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_pose_process.json new file mode 100644 index 0000000..2893dd2 --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_t2i_control_pose_process.json @@ -0,0 +1,692 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 104, + "last_link_id": 99, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 91, + "type": "LoadZImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTextEncoderModel" + }, + "widgets_values": [ + "qwen_3_4b.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 88 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 97, + "type": "Note", + "pos": [ + -354.4680507215508, + -433.6570714778354 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 96, + "type": "CombineZImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 96 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "CombineZImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload" + ] + }, + { + "id": 92, + "type": "LoadZImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 95 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageTransformerModel" + }, + "widgets_values": [ + "z_image_turbo_bf16.safetensors", + "bf16" + ] + }, + { + "id": 102, + "type": "LoadZImageControlNetInModel", + "pos": [ + 779.793189390101, + -457.3825558553134 + ], + "size": [ + 589.8698159570357, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 95 + } + ], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 96 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageControlNetInModel" + }, + "widgets_values": [ + "z_image/z_image_control_2.1.yaml", + "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" + ] + }, + { + "id": 99, + "type": "ZImageControlSampler", + "pos": [ + 727.2482831521481, + -52.41010988674983 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 87 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 88 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "control_image", + "shape": 7, + "type": "IMAGE", + "link": 98 + }, + { + "name": "inpaint_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "mask_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageControlSampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3, + 0.8 + ] + }, + { + "id": 93, + "type": "LoadZImageVAEModel", + "pos": [ + 1168.1599508804559, + -314.8650476692169 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageVAEModel" + }, + "widgets_values": [ + "ae.safetensors", + "bf16" + ] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 281.12436800744575, + 417.89046761218935 + ], + "size": [ + 270, + 314.00000000000006 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 97 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "z-image-turbo_00004_.png", + "image" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1049.1402001998824, + -52.699751832945104 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 90 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 103, + "type": "ImageToPose", + "pos": [ + 582.6473087858849, + 424.6900692435995 + ], + "size": [ + 226.09371582945823, + 27.59660529340124 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "input_image", + "type": "IMAGE", + "link": 97 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 98, + 99 + ] + } + ], + "properties": { + "Node name for S&R": "ImageToPose" + } + }, + { + "id": 104, + "type": "PreviewImage", + "pos": [ + 844.9628845788926, + 422.400403774606 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 99 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + } + ], + "links": [ + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 87, + 96, + 0, + 99, + 0, + "FunModels" + ], + [ + 88, + 75, + 0, + 99, + 1, + "STRING_PROMPT" + ], + [ + 89, + 73, + 0, + 99, + 2, + "STRING_PROMPT" + ], + [ + 90, + 99, + 0, + 88, + 0, + "IMAGE" + ], + [ + 95, + 92, + 0, + 102, + 0, + "TransformerModel" + ], + [ + 96, + 102, + 0, + 96, + 0, + "TransformerModel" + ], + [ + 97, + 100, + 0, + 103, + 0, + "IMAGE" + ], + [ + 98, + 103, + 0, + 99, + 3, + "IMAGE" + ], + [ + 99, + 103, + 0, + 104, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 1350.4793699732413, + 404.87677206390265 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7608917213421558, + "offset": [ + 217.08766227758116, + 385.1943830579828 + ] + }, + "frontendVersion": "1.36.11", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "93aa7b2530dccd1e91c625eee439a5e24f8ffa04", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_tile.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_tile.json similarity index 100% rename from comfyui/z_image/v1/z_image_chunked_loading_workflow_tile.json rename to comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_tile.json diff --git a/comfyui/z_image/v1/z_image_chunked_loading_workflow_tile_lite.json b/comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_tile_lite.json similarity index 100% rename from comfyui/z_image/v1/z_image_chunked_loading_workflow_tile_lite.json rename to comfyui/z_image/v1/z_image_turbo_chunked_loading_workflow_tile_lite.json diff --git a/comfyui/z_image/v1/z_image_turbo_workflow_t2i.json b/comfyui/z_image/v1/z_image_turbo_workflow_t2i.json new file mode 100644 index 0000000..191d52e --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_workflow_t2i.json @@ -0,0 +1,320 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 91, + "last_link_id": 70, + "nodes": [ + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 69 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 68 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1070.207763671875, + -73.63389587402344 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 70 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -424.31362772254084, + -372.4056987182365 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 86, + "type": "LoadZImageModel", + "pos": [ + 241.10867359909312, + -295.7069265122969 + ], + "size": [ + 428.9360739181402, + 120.18527437036698 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageModel" + }, + "widgets_values": [ + "Z-Image-Turbo", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 91, + "type": "ZImageT2ISampler", + "pos": [ + 719.3201904296875, + -72.24609375 + ], + "size": [ + 280.724609375, + 386 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 67 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 68 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 69 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 70 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageT2ISampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3 + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 17.042007499433538, + -27.473822111441212 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 67, + 86, + 0, + 91, + 0, + "FunModels" + ], + [ + 68, + 75, + 0, + 91, + 1, + "STRING_PROMPT" + ], + [ + 69, + 73, + 0, + 91, + 2, + "STRING_PROMPT" + ], + [ + 70, + 91, + 0, + 88, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6378658119731967, + "offset": [ + 856.870045195949, + 740.927030543633 + ] + }, + "frontendVersion": "1.36.11", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "244f11053106af1a58ac25fd0bbb508fd7b89c0f", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_turbo_workflow_t2i_control.json b/comfyui/z_image/v1/z_image_turbo_workflow_t2i_control.json new file mode 100644 index 0000000..cff8ce5 --- /dev/null +++ b/comfyui/z_image/v1/z_image_turbo_workflow_t2i_control.json @@ -0,0 +1,431 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 101, + "last_link_id": 92, + "nodes": [ + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 90 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 17.042007499433538, + -27.473822111441212 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A photo of Sakura, a 17-year-old high school student from Japan, captured in a candid, high-fidelity cinematic moment on a rainy evening. She is squatting low on the rain-slicked asphalt of an urban sidewalk, holding a transparent vinyl umbrella with a white handle resting over her shoulder in one hand, her other hand resting on her knee. The clear plastic canopy is streaked with rivulets of water and beaded with droplets that catch the ambient city light. A profound, silent interaction defines the scene: Sakura is looking directly downward, her expression gentle and focused, locking eyes with a small black cat sitting on the wet ground in front of her.\n\nSakura has long, lustrous black hair styled in a precise hime cut with blunt bangs across her forehead and sidelocks framing her cheeks, damp strands clinging subtly to her jacket, with a single red ribbon tied on the left side. Her visible pores on her nose, and a soft sheen of moisture on her cheeks. She wears a dark navy sailor-style school uniform (seifuku) featuring a white collar with red linear detailing and a bright red necktie loosely knotted at the chest; a simple black choker encircles her neck. The uniform jacket has oversized sleeves. Her lower body features a short, dark pleated miniskirt that fans slightly over clean white ankle socks that provide a stark contrast to the wet asphalt, ending in dark leather loafers that gleam with moisture.\n\nThe black cat sits upright in a shallow puddle, its short fur slicked by the rain, tilting its head back to stare intently up into Sakura's face, establishing a clear line of sight. The background is anchored by a large, illuminated red vending machine standing against the darkness, its cool bluish-white interior light spilling onto Sakura's profile and the umbrella. The ground reflects the red chassis and the neon streetlights in distorted patches on the wet pavement. Additional cool rain streaks fall through the frame, some caught in sharp focus and others blurred into vertical lines against the background lights. The scene is rendered with a wide-aperture lens creating a shallow depth of field, keeping the girl and cat in sharp focus while softening the background into gentle bokeh, with the texture of fine-grain 35mm film stock.\n" + ] + }, + { + "id": 96, + "type": "LoadImage", + "pos": [ + 429.45065831038295, + 448.4639992924031 + ], + "size": [ + 270, + 314 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 91 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "a7kXeQ5l9Dhspes7q3x3G (1).png", + "image" + ] + }, + { + "id": 86, + "type": "LoadZImageModel", + "pos": [ + 158.6376150119142, + -352.0410222978062 + ], + "size": [ + 428.9360739181402, + 120.18527437036698 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageModel" + }, + "widgets_values": [ + "Z-Image-Turbo", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -486.1347709940621, + -416.6940105696127 + ], + "size": [ + 598.1623727144193, + 233.48501180428053 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload].\nmodel_full_load means that the entire model will be moved to the GPU.\n\nmodel_full_load_and_qfloat8 means that the entire model will be moved to the GPU,\nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nmodel_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory.\n\nmodel_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, \nand the transformer model has been quantized to float8, which can save more GPU memory. \n\nsequential_cpu_offload means that each layer of the model will be moved to the CPU after use, \nresulting in slower speeds but saving a large amount of GPU memory." + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 99, + "type": "LoadZImageControlNetInPipeline", + "pos": [ + 662.479156335491, + -343.8489854292294 + ], + "size": [ + 556.7720385739738, + 107.30044919237525 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 81 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 88 + ] + } + ], + "properties": { + "Node name for S&R": "LoadZImageControlNetInPipeline" + }, + "widgets_values": [ + "z_image/z_image_control_2.1.yaml", + "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors", + "transformer" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1070.207763671875, + -73.63389587402344 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 92 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 101, + "type": "ZImageControlSampler", + "pos": [ + 752.6720492832281, + -61.24351896170549 + ], + "size": [ + 270, + 350 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 88 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 90 + }, + { + "name": "control_image", + "shape": 7, + "type": "IMAGE", + "link": 91 + }, + { + "name": "inpaint_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "mask_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 92 + ] + } + ], + "properties": { + "Node name for S&R": "ZImageControlSampler" + }, + "widgets_values": [ + 1184, + 1568, + 43, + "fixed", + 8, + 0, + "Flow", + 3, + 0.8 + ] + } + ], + "links": [ + [ + 81, + 86, + 0, + 99, + 0, + "FunModels" + ], + [ + 88, + 99, + 0, + 101, + 0, + "FunModels" + ], + [ + 89, + 75, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 90, + 73, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 91, + 96, + 0, + 101, + 3, + "IMAGE" + ], + [ + 92, + 101, + 0, + 88, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 137.52894141282107, + -436.3340957855093, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7396517503305734, + "offset": [ + 584.364342724736, + 515.7058946883703 + ] + }, + "frontendVersion": "1.36.11", + "workflowRendererVersion": "LG", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "244f11053106af1a58ac25fd0bbb508fd7b89c0f", + "comfy-core": "0.6.0" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/comfyui/z_image/v1/z_image_workflow_t2i.json b/comfyui/z_image/v1/z_image_workflow_t2i.json index 191d52e..7b9d9c8 100644 --- a/comfyui/z_image/v1/z_image_workflow_t2i.json +++ b/comfyui/z_image/v1/z_image_workflow_t2i.json @@ -34,7 +34,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -150,8 +150,8 @@ "Node name for S&R": "LoadZImageModel" }, "widgets_values": [ - "Z-Image-Turbo", - "model_cpu_offload", + "Z-Image", + "model_group_offload", "bf16" ] }, diff --git a/comfyui/z_image/v1/z_image_workflow_t2i_control.json b/comfyui/z_image/v1/z_image_workflow_t2i_control.json index cff8ce5..cd751b8 100644 --- a/comfyui/z_image/v1/z_image_workflow_t2i_control.json +++ b/comfyui/z_image/v1/z_image_workflow_t2i_control.json @@ -34,7 +34,7 @@ "Node name for S&R": "FunTextBox" }, "widgets_values": [ - "" + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。" ] }, { @@ -160,8 +160,8 @@ "Node name for S&R": "LoadZImageModel" }, "widgets_values": [ - "Z-Image-Turbo", - "model_cpu_offload", + "Z-Image", + "model_group_offload", "bf16" ] }, @@ -225,7 +225,7 @@ }, "widgets_values": [ "z_image/z_image_control_2.1.yaml", - "Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors", + "Z-Image-Fun-Controlnet-Union-2.1.safetensors", "transformer" ] }, diff --git a/examples/flux2_fun/predict_i2i_inpaint.py b/examples/flux2_fun/predict_i2i_inpaint.py index 8b42e99..5c0df8d 100644 --- a/examples/flux2_fun/predict_i2i_inpaint.py +++ b/examples/flux2_fun/predict_i2i_inpaint.py @@ -69,7 +69,7 @@ model_name = "models/Diffusion_Transformer/FLUX.2-dev" sampler_name = "Flow" # Load pretrained model if need -transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" +transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" vae_path = None lora_path = None diff --git a/examples/flux2_fun/predict_t2i_control.py b/examples/flux2_fun/predict_t2i_control.py index 869f0b7..6dd9740 100644 --- a/examples/flux2_fun/predict_t2i_control.py +++ b/examples/flux2_fun/predict_t2i_control.py @@ -69,7 +69,7 @@ model_name = "models/Diffusion_Transformer/FLUX.2-dev" sampler_name = "Flow" # Load pretrained model if need -transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" +transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" vae_path = None lora_path = None diff --git a/examples/flux2_fun/predict_t2i_control_ref.py b/examples/flux2_fun/predict_t2i_control_ref.py index 5687b96..106b525 100644 --- a/examples/flux2_fun/predict_t2i_control_ref.py +++ b/examples/flux2_fun/predict_t2i_control_ref.py @@ -69,7 +69,7 @@ model_name = "models/Diffusion_Transformer/FLUX.2-dev" sampler_name = "Flow" # Load pretrained model if need -transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" +transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" vae_path = None lora_path = None diff --git a/examples/z_image/predict_t2i.py b/examples/z_image/predict_t2i.py index 4fe635a..870be5a 100644 --- a/examples/z_image/predict_t2i.py +++ b/examples/z_image/predict_t2i.py @@ -160,13 +160,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_i2i_inpaint_2.1.py b/examples/z_image_fun/predict_i2i_inpaint_2.1.py index 78336ae..e9249aa 100644 --- a/examples/z_image_fun/predict_i2i_inpaint_2.1.py +++ b/examples/z_image_fun/predict_i2i_inpaint_2.1.py @@ -56,13 +56,13 @@ compile_dit = False # Config and model path config_path = "config/z_image/z_image_control_2.1.yaml" # model path -model_name = "models/Diffusion_Transformer/Z-Image-Turbo" +model_name = "models/Diffusion_Transformer/Z-Image" # Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" sampler_name = "Flow" # Load pretrained model if need -transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" +transformer_path = "models/Personalized_Model/Z-Image-Fun-Controlnet-Union-2.1-8steps.safetensors" vae_path = None lora_path = None @@ -175,13 +175,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_i2i_inpaint_2.1_lite.py b/examples/z_image_fun/predict_i2i_inpaint_2.1_lite.py index 4c30ab8..a150687 100644 --- a/examples/z_image_fun/predict_i2i_inpaint_2.1_lite.py +++ b/examples/z_image_fun/predict_i2i_inpaint_2.1_lite.py @@ -56,13 +56,13 @@ compile_dit = False # Config and model path config_path = "config/z_image/z_image_control_2.1_lite.yaml" # model path -model_name = "models/Diffusion_Transformer/Z-Image-Turbo" +model_name = "models/Diffusion_Transformer/Z-Image" # Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" sampler_name = "Flow" # Load pretrained model if need -transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" +transformer_path = "models/Personalized_Model/Z-Image-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" vae_path = None lora_path = None @@ -175,13 +175,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_t2i_control_2.1.py b/examples/z_image_fun/predict_t2i_control_2.1.py index c8dd8cb..6d99b2c 100644 --- a/examples/z_image_fun/predict_t2i_control_2.1.py +++ b/examples/z_image_fun/predict_t2i_control_2.1.py @@ -56,13 +56,13 @@ compile_dit = False # Config and model path config_path = "config/z_image/z_image_control_2.1.yaml" # model path -model_name = "models/Diffusion_Transformer/Z-Image-Turbo" +model_name = "models/Diffusion_Transformer/Z-Image" # Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" sampler_name = "Flow" # Load pretrained model if need -transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" +transformer_path = "models/Personalized_Model/Z-Image-Fun-Controlnet-Union-2.1-8steps.safetensors" vae_path = None lora_path = None @@ -175,13 +175,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_t2i_control_2.1_lite.py b/examples/z_image_fun/predict_t2i_control_2.1_lite.py index 2aa9726..5fa21ad 100644 --- a/examples/z_image_fun/predict_t2i_control_2.1_lite.py +++ b/examples/z_image_fun/predict_t2i_control_2.1_lite.py @@ -56,13 +56,13 @@ compile_dit = False # Config and model path config_path = "config/z_image/z_image_control_2.1_lite.yaml" # model path -model_name = "models/Diffusion_Transformer/Z-Image-Turbo" +model_name = "models/Diffusion_Transformer/Z-Image" # Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" sampler_name = "Flow" # Load pretrained model if need -transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" +transformer_path = "models/Personalized_Model/Z-Image-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" vae_path = None lora_path = None @@ -175,13 +175,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_i2i_inpaint_2.0.py b/examples/z_image_fun/predict_turbo_i2i_inpaint_2.0.py similarity index 99% rename from examples/z_image_fun/predict_i2i_inpaint_2.0.py rename to examples/z_image_fun/predict_turbo_i2i_inpaint_2.0.py index f768402..e9a3b5d 100644 --- a/examples/z_image_fun/predict_i2i_inpaint_2.0.py +++ b/examples/z_image_fun/predict_turbo_i2i_inpaint_2.0.py @@ -175,13 +175,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_turbo_i2i_inpaint_2.1.py b/examples/z_image_fun/predict_turbo_i2i_inpaint_2.1.py new file mode 100644 index 0000000..5cef224 --- /dev/null +++ b/examples/z_image_fun/predict_turbo_i2i_inpaint_2.1.py @@ -0,0 +1,241 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import ZImageControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, get_image, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/z_image/z_image_control_2.1.yaml" +# model path +model_name = "models/Diffusion_Transformer/Z-Image-Turbo" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_image = "asset/pose.jpg" +inpaint_image = "asset/8.png" +mask_image = "asset/mask.png" +control_context_scale = 0.75 + +# Please use as detailed a prompt as possible to describe the object that needs to be generated. +prompt = "画面中央是一位年轻女孩，她拥有一头令人印象深刻的亮紫色长发，发丝在海风中轻盈飘扬，营造出动感而唯美的效果。她的长发两侧各扎着黑色蝴蝶结发饰，增添了几分可爱与俏皮感。女孩身穿一袭纯白色无袖连衣裙，裙摆轻盈飘逸，与她清新的气质完美契合。她的妆容精致自然，淡粉色的唇妆和温柔的眼神流露出恬静优雅的气质。她单手叉腰，姿态自信从容，目光直视镜头，展现出既甜美又不失个性的魅力。背景是一片开阔的海景，湛蓝的海水在阳光照射下波光粼粼，闪烁着钻石般的光芒。天空呈现出清澈的蔚蓝色，点缀着几朵洁白的云朵，营造出晴朗明媚的夏日氛围。画面前景右下角可见粉紫色的小花丛和绿色植物，为整体构图增添了自然生机和色彩层次。整张照片色调明亮清新，紫色头发与白色裙装、蓝色海天形成鲜明而和谐的色彩对比，呈现出一种童话般的浪漫意境，宛如二次元世界与现实海景的完美融合。" +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 8 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = ZImageControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = ZImageControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.layers)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(text_encoder.model.layers)) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if inpaint_image is not None: + inpaint_image = get_image_latent(inpaint_image, sample_size=sample_size)[:, :, 0] + else: + inpaint_image = torch.zeros([1, 3, sample_size[0], sample_size[1]]) + + if mask_image is not None: + mask_image = get_image_latent(mask_image, sample_size=sample_size)[:, :1, 0] + else: + mask_image = torch.ones([1, 1, sample_size[0], sample_size[1]]) * 255 + + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] + + sample = pipeline( + prompt = prompt, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + image = inpaint_image, + mask_image = mask_image, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/examples/z_image_fun/predict_turbo_i2i_inpaint_2.1_lite.py b/examples/z_image_fun/predict_turbo_i2i_inpaint_2.1_lite.py new file mode 100644 index 0000000..6b042cc --- /dev/null +++ b/examples/z_image_fun/predict_turbo_i2i_inpaint_2.1_lite.py @@ -0,0 +1,241 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import ZImageControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, get_image, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/z_image/z_image_control_2.1_lite.yaml" +# model path +model_name = "models/Diffusion_Transformer/Z-Image-Turbo" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_image = "asset/pose.jpg" +inpaint_image = "asset/8.png" +mask_image = "asset/mask.png" +control_context_scale = 0.85 + +# Please use as detailed a prompt as possible to describe the object that needs to be generated. +prompt = "画面中央是一位年轻女孩，她拥有一头令人印象深刻的亮紫色长发，发丝在海风中轻盈飘扬，营造出动感而唯美的效果。她的长发两侧各扎着黑色蝴蝶结发饰，增添了几分可爱与俏皮感。女孩身穿一袭纯白色无袖连衣裙，裙摆轻盈飘逸，与她清新的气质完美契合。她的妆容精致自然，淡粉色的唇妆和温柔的眼神流露出恬静优雅的气质。她单手叉腰，姿态自信从容，目光直视镜头，展现出既甜美又不失个性的魅力。背景是一片开阔的海景，湛蓝的海水在阳光照射下波光粼粼，闪烁着钻石般的光芒。天空呈现出清澈的蔚蓝色，点缀着几朵洁白的云朵，营造出晴朗明媚的夏日氛围。画面前景右下角可见粉紫色的小花丛和绿色植物，为整体构图增添了自然生机和色彩层次。整张照片色调明亮清新，紫色头发与白色裙装、蓝色海天形成鲜明而和谐的色彩对比，呈现出一种童话般的浪漫意境，宛如二次元世界与现实海景的完美融合。" +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 8 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = ZImageControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = ZImageControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.layers)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(text_encoder.model.layers)) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if inpaint_image is not None: + inpaint_image = get_image_latent(inpaint_image, sample_size=sample_size)[:, :, 0] + else: + inpaint_image = torch.zeros([1, 3, sample_size[0], sample_size[1]]) + + if mask_image is not None: + mask_image = get_image_latent(mask_image, sample_size=sample_size)[:, :1, 0] + else: + mask_image = torch.ones([1, 1, sample_size[0], sample_size[1]]) * 255 + + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] + + sample = pipeline( + prompt = prompt, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + image = inpaint_image, + mask_image = mask_image, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/examples/z_image_fun/predict_i2i_tile_2.1.py b/examples/z_image_fun/predict_turbo_i2i_tile_2.1.py similarity index 99% rename from examples/z_image_fun/predict_i2i_tile_2.1.py rename to examples/z_image_fun/predict_turbo_i2i_tile_2.1.py index 8ed2616..c9d74b7 100644 --- a/examples/z_image_fun/predict_i2i_tile_2.1.py +++ b/examples/z_image_fun/predict_turbo_i2i_tile_2.1.py @@ -176,13 +176,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_i2i_tile_2.1_lite.py b/examples/z_image_fun/predict_turbo_i2i_tile_2.1_lite.py similarity index 99% rename from examples/z_image_fun/predict_i2i_tile_2.1_lite.py rename to examples/z_image_fun/predict_turbo_i2i_tile_2.1_lite.py index d8e2eb0..497bc7c 100644 --- a/examples/z_image_fun/predict_i2i_tile_2.1_lite.py +++ b/examples/z_image_fun/predict_turbo_i2i_tile_2.1_lite.py @@ -176,13 +176,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_t2i_control.py b/examples/z_image_fun/predict_turbo_t2i_control.py similarity index 98% rename from examples/z_image_fun/predict_t2i_control.py rename to examples/z_image_fun/predict_turbo_t2i_control.py index b1a9ebf..b561e8b 100644 --- a/examples/z_image_fun/predict_t2i_control.py +++ b/examples/z_image_fun/predict_turbo_t2i_control.py @@ -173,13 +173,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_t2i_control_2.0.py b/examples/z_image_fun/predict_turbo_t2i_control_2.0.py similarity index 99% rename from examples/z_image_fun/predict_t2i_control_2.0.py rename to examples/z_image_fun/predict_turbo_t2i_control_2.0.py index a861395..a9d38d9 100644 --- a/examples/z_image_fun/predict_t2i_control_2.0.py +++ b/examples/z_image_fun/predict_turbo_t2i_control_2.0.py @@ -175,13 +175,13 @@ if compile_dit: if GPU_memory_mode == "sequential_cpu_offload": pipeline.enable_sequential_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_cpu_offload": pipeline.enable_model_cpu_offload(device=device) elif GPU_memory_mode == "model_full_load_and_qfloat8": - convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) convert_weight_dtype_wrapper(transformer, weight_dtype) pipeline.to(device=device) else: diff --git a/examples/z_image_fun/predict_turbo_t2i_control_2.1.py b/examples/z_image_fun/predict_turbo_t2i_control_2.1.py new file mode 100644 index 0000000..b850014 --- /dev/null +++ b/examples/z_image_fun/predict_turbo_t2i_control_2.1.py @@ -0,0 +1,241 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import ZImageControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, get_image, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/z_image/z_image_control_2.1.yaml" +# model path +model_name = "models/Diffusion_Transformer/Z-Image-Turbo" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1-8steps.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_image = "asset/pose.jpg" +inpaint_image = None +mask_image = None +control_context_scale = 0.75 + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +prompt = "画面中央是一位年轻女孩，她拥有一头令人印象深刻的亮紫色长发，发丝在海风中轻盈飘扬，营造出动感而唯美的效果。她的长发两侧各扎着黑色蝴蝶结发饰，增添了几分可爱与俏皮感。女孩身穿一袭纯白色无袖连衣裙，裙摆轻盈飘逸，与她清新的气质完美契合。她的妆容精致自然，淡粉色的唇妆和温柔的眼神流露出恬静优雅的气质。她单手叉腰，姿态自信从容，目光直视镜头，展现出既甜美又不失个性的魅力。背景是一片开阔的海景，湛蓝的海水在阳光照射下波光粼粼，闪烁着钻石般的光芒。天空呈现出清澈的蔚蓝色，点缀着几朵洁白的云朵，营造出晴朗明媚的夏日氛围。画面前景右下角可见粉紫色的小花丛和绿色植物，为整体构图增添了自然生机和色彩层次。整张照片色调明亮清新，紫色头发与白色裙装、蓝色海天形成鲜明而和谐的色彩对比，呈现出一种童话般的浪漫意境，宛如二次元世界与现实海景的完美融合。" +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 8 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = ZImageControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = ZImageControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.layers)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(text_encoder.model.layers)) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if inpaint_image is not None: + inpaint_image = get_image_latent(inpaint_image, sample_size=sample_size)[:, :, 0] + else: + inpaint_image = torch.zeros([1, 3, sample_size[0], sample_size[1]]) + + if mask_image is not None: + mask_image = get_image_latent(mask_image, sample_size=sample_size)[:, :1, 0] + else: + mask_image = torch.ones([1, 1, sample_size[0], sample_size[1]]) * 255 + + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] + + sample = pipeline( + prompt = prompt, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + image = inpaint_image, + mask_image = mask_image, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/examples/z_image_fun/predict_turbo_t2i_control_2.1_lite.py b/examples/z_image_fun/predict_turbo_t2i_control_2.1_lite.py new file mode 100644 index 0000000..50179e4 --- /dev/null +++ b/examples/z_image_fun/predict_turbo_t2i_control_2.1_lite.py @@ -0,0 +1,241 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import ZImageControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, get_image, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/z_image/z_image_control_2.1_lite.yaml" +# model path +model_name = "models/Diffusion_Transformer/Z-Image-Turbo" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1-lite-2601-8steps.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_image = "asset/pose.jpg" +inpaint_image = None +mask_image = None +control_context_scale = 0.85 + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +prompt = "画面中央是一位年轻女孩，她拥有一头令人印象深刻的亮紫色长发，发丝在海风中轻盈飘扬，营造出动感而唯美的效果。她的长发两侧各扎着黑色蝴蝶结发饰，增添了几分可爱与俏皮感。女孩身穿一袭纯白色无袖连衣裙，裙摆轻盈飘逸，与她清新的气质完美契合。她的妆容精致自然，淡粉色的唇妆和温柔的眼神流露出恬静优雅的气质。她单手叉腰，姿态自信从容，目光直视镜头，展现出既甜美又不失个性的魅力。背景是一片开阔的海景，湛蓝的海水在阳光照射下波光粼粼，闪烁着钻石般的光芒。天空呈现出清澈的蔚蓝色，点缀着几朵洁白的云朵，营造出晴朗明媚的夏日氛围。画面前景右下角可见粉紫色的小花丛和绿色植物，为整体构图增添了自然生机和色彩层次。整张照片色调明亮清新，紫色头发与白色裙装、蓝色海天形成鲜明而和谐的色彩对比，呈现出一种童话般的浪漫意境，宛如二次元世界与现实海景的完美融合。" +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 8 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = ZImageControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = ZImageControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.layers)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(text_encoder.model.layers)) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_pad_token", "cap_pad_token"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if inpaint_image is not None: + inpaint_image = get_image_latent(inpaint_image, sample_size=sample_size)[:, :, 0] + else: + inpaint_image = torch.zeros([1, 3, sample_size[0], sample_size[1]]) + + if mask_image is not None: + mask_image = get_image_latent(mask_image, sample_size=sample_size)[:, :1, 0] + else: + mask_image = torch.ones([1, 1, sample_size[0], sample_size[1]]) * 255 + + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] + + sample = pipeline( + prompt = prompt, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + image = inpaint_image, + mask_image = mask_image, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/scripts/flux/train_lora.py b/scripts/flux/train_lora.py index c6dae26..f9b7811 100644 --- a/scripts/flux/train_lora.py +++ b/scripts/flux/train_lora.py @@ -1025,9 +1025,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/flux2/train_lora.py b/scripts/flux2/train_lora.py index 2469bfe..4405278 100644 --- a/scripts/flux2/train_lora.py +++ b/scripts/flux2/train_lora.py @@ -1086,9 +1086,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/flux2_fun/README_TRAIN.md b/scripts/flux2_fun/README_TRAIN.md index a79d9a0..950e14f 100644 --- a/scripts/flux2_fun/README_TRAIN.md +++ b/scripts/flux2_fun/README_TRAIN.md @@ -71,7 +71,7 @@ accelerate launch --mixed_precision="bf16" scripts/flux2_fun/train_control.py \ --enable_bucket \ --low_vram \ --uniform_sampling \ - --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" \ + --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" \ --trainable_modules "control" \ --resume_from_checkpoint="latest" ``` @@ -112,7 +112,7 @@ accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_con --enable_bucket \ --low_vram \ --uniform_sampling \ - --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" \ + --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" \ --trainable_modules "control" \ --resume_from_checkpoint="latest" ``` @@ -153,7 +153,7 @@ accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TR --enable_bucket \ --low_vram \ --uniform_sampling \ - --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" \ + --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" \ --trainable_modules "control" \ --resume_from_checkpoint="latest" ``` \ No newline at end of file diff --git a/scripts/flux2_fun/train_control.sh b/scripts/flux2_fun/train_control.sh index 48d0c05..cc72759 100644 --- a/scripts/flux2_fun/train_control.sh +++ b/scripts/flux2_fun/train_control.sh @@ -31,6 +31,6 @@ accelerate launch --mixed_precision="bf16" scripts/flux2_fun/train_control.py \ --enable_bucket \ --low_vram \ --uniform_sampling \ - --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" \ + --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" \ --trainable_modules "control" \ --resume_from_checkpoint="latest" \ No newline at end of file diff --git a/scripts/flux2_fun/train_control_distill.py b/scripts/flux2_fun/train_control_distill.py new file mode 100644 index 0000000..7c6b869 --- /dev/null +++ b/scripts/flux2_fun/train_control_distill.py @@ -0,0 +1,1898 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +import argparse +import contextlib +import gc +import json +import logging +import math +import os +import pickle +import random +import shutil +import sys +from typing import (Any, Callable, Dict, List, NamedTuple, Optional, Tuple, + Union) + +import accelerate +import diffusers +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import Dataset, RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from qwen_vl_utils import process_vision_info + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + process_pose_file, + process_pose_params) +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLFlux2, AutoProcessor, + Flux2ControlTransformer2DModel, + Mistral3ForConditionalGeneration, + PixtralProcessor) +from videox_fun.pipeline import Flux2ControlPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import (calculate_dimensions, get_image_latent, + get_image_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +def _prepare_latent_ids( + latents: torch.Tensor, # (B, C, H, W) +): + r""" + Generates 4D position coordinates (T, H, W, L) for latent tensors. + + Args: + latents (torch.Tensor): + Latent tensor of shape (B, C, H, W) + + Returns: + torch.Tensor: + Position IDs tensor of shape (B, H*W, 4) All batches share the same coordinate structure: T=0, + H=[0..H-1], W=[0..W-1], L=0 + """ + + batch_size, _, height, width = latents.shape + + t = torch.arange(1) # [0] - time dimension + h = torch.arange(height) + w = torch.arange(width) + l = torch.arange(1) # [0] - layer dimension + + # Create position IDs: (H*W, 4) + latent_ids = torch.cartesian_prod(t, h, w, l) + + # Expand to batch: (B, H*W, 4) + latent_ids = latent_ids.unsqueeze(0).expand(batch_size, -1, -1) + + return latent_ids + +def _patchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 1, 3, 5, 2, 4) + latents = latents.reshape(batch_size, num_channels_latents * 4, height // 2, width // 2) + return latents + +def _pack_latents(latents): + """ + pack latents: (batch_size, num_channels, height, width) -> (batch_size, height * width, num_channels) + """ + + batch_size, num_channels, height, width = latents.shape + latents = latents.reshape(batch_size, num_channels, height * width).permute(0, 2, 1) + + return latents + +def format_text_input(prompts: List[str], system_message: str = None): + # Remove [IMG] tokens from prompts to avoid Pixtral validation issues + # when truncation is enabled. The processor counts [IMG] tokens and fails + # if the count changes after truncation. + cleaned_txt = [prompt.replace("[IMG]", "") for prompt in prompts] + + return [ + [ + { + "role": "system", + "content": [{"type": "text", "text": system_message}], + }, + {"role": "user", "content": [{"type": "text", "text": prompt}]}, + ] + for prompt in cleaned_txt + ] + +def _get_mistral_3_small_prompt_embeds( + text_encoder: Mistral3ForConditionalGeneration, + tokenizer: PixtralProcessor, + prompt: Union[str, List[str]], + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + max_sequence_length: int = 512, + # fmt: off + system_message: str = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation.", + # fmt: on + hidden_states_layers: List[int] = (10, 20, 30), +): + dtype = text_encoder.dtype if dtype is None else dtype + device = text_encoder.device if device is None else device + + prompt = [prompt] if isinstance(prompt, str) else prompt + + # Format input messages + messages_batch = format_text_input(prompts=prompt, system_message=system_message) + + # Process all messages at once + inputs = tokenizer.apply_chat_template( + messages_batch, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=max_sequence_length, + ) + + # Move to device + input_ids = inputs["input_ids"].to(device) + attention_mask = inputs["attention_mask"].to(device) + + # Forward pass through the model + output = text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + use_cache=False, + ) + + # Only use outputs from intermediate layers and stack them + out = torch.stack([output.hidden_states[k] for k in hidden_states_layers], dim=1) + out = out.to(dtype=dtype, device=device) + + batch_size, num_channels, seq_len, hidden_dim = out.shape + prompt_embeds = out.permute(0, 2, 1, 3).reshape(batch_size, seq_len, num_channels * hidden_dim) + + return prompt_embeds + +def _prepare_text_ids( + x: torch.Tensor, # (B, L, D) or (L, D) + t_coord: Optional[torch.Tensor] = None, +): + B, L, _ = x.shape + out_ids = [] + + for i in range(B): + t = torch.arange(1) if t_coord is None else t_coord[i] + h = torch.arange(1) + w = torch.arange(1) + l = torch.arange(L) + + coords = torch.cartesian_prod(t, h, w, l) + out_ids.append(coords) + + return torch.stack(out_ids) + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + text_encoder=None, + tokenizer=None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + text_encoder_out_layers: Tuple[int] = (10, 20, 30), + system_message = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation." +): + if prompt is None: + prompt = "" + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt_embeds is None: + prompt_embeds = _get_mistral_3_small_prompt_embeds( + text_encoder=text_encoder, + tokenizer=tokenizer, + prompt=prompt, + device=device, + max_sequence_length=max_sequence_length, + system_message=system_message, + hidden_states_layers=text_encoder_out_layers, + ) + + batch_size, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + text_ids = _prepare_text_ids(prompt_embeds) + text_ids = text_ids.to(device) + return prompt_embeds, text_ids + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, accelerator, weight_dtype, global_step): + try: + is_deepspeed = type(transformer3d).__name__ == 'DeepSpeedEngine' + if is_deepspeed: + origin_config = transformer3d.config + transformer3d.config = accelerator.unwrap_model(transformer3d).config + with torch.no_grad(), torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + logger.info("Running validation... ") + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + pipeline = Flux2ControlPipeline( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + transformer=transformer3d, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + rank_seed = args.seed + accelerator.process_index + generator = torch.Generator(device=accelerator.device).manual_seed(rank_seed) + logger.info(f"Rank {accelerator.process_index} using seed: {rank_seed}") + + for i in range(len(args.validation_prompts)): + control_image = Image.open(args.validation_paths[i]) + width, height = control_image.width, control_image.height + width, height = calculate_dimensions(args.image_sample_size * args.image_sample_size, width / height) + control_image = get_image_latent(control_image, sample_size=(height, width))[:, :, 0] + + sample = pipeline( + prompt = args.validation_prompts[i], + height = height, + width = width, + generator = generator, + num_inference_steps = 20, + control_context_scale = 0.90, + control_image = control_image, + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save( + os.path.join( + args.output_dir, + f"sample/sample-{global_step}-rank{accelerator.process_index}-image-{i}.jpg" + ) + ) + + del pipeline + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if is_deepspeed: + transformer3d.config = origin_config + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error on rank {accelerator.process_index} with info {e}") + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--prompt_template_encode", + type=str, + default="<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", + help=( + 'The prompt template for text encoder.' + ), + ) + parser.add_argument( + "--prompt_template_encode_start_idx", + type=int, + default=34, + help=( + 'The start idx for prompt template.' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=3.5, + help="the FLUX.1 dev variant is a guidance distilled model", + ) + parser.add_argument( + "--real_guidance_scale", + type=float, + default=4.0, + help="The cfg scale for real score.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = PixtralProcessor.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + config = OmegaConf.load(args.config_path) + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So Mistral3ForConditionalGeneration and AutoencoderKLFlux2 will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Mistral3ForConditionalGeneration.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLFlux2.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + latents_bn_mean = vae.bn.running_mean.view(1, -1, 1, 1).to(accelerator.device, weight_dtype) + latents_bn_std = torch.sqrt(vae.bn.running_var.view(1, -1, 1, 1) + vae.config.batch_norm_eps).to(accelerator.device, weight_dtype) + + # Get Transformer + generator_transformer3d = Flux2ControlTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + real_score_transformer3d = Flux2ControlTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + generator_transformer3d.requires_grad_(False) + real_score_transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = generator_transformer3d.load_state_dict(state_dict, strict=False) + m, u = real_score_transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + generator_transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in generator_transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = Flux2ControlTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + generator_transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, generator_transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in generator_transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_inpaint=True, + enable_camera_info=False, + enable_subject_info=False, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + return [first_element] + [other_elements_value] * (length - 1) + + MIN_TARGET = 1024 + + if sample_size < MIN_TARGET: + number_list = [1.0] + else: + max_allowed_ratio = sample_size / MIN_TARGET + base_ratios = [ + 1.0, + 1.1, 1.2, 1.25, 1.33, 1.5, + 1.75, 2.0, 2.25, 2.5, 2.75, + 3.0, 3.5, 4.0, 5.0, 6.0, 8.0 + ] + candidate_ratios = set(base_ratios + list(image_ratio)) + number_list = sorted([r for r in candidate_ratios if 1.0 <= r <= max_allowed_ratio]) + + if not number_list: + number_list = [1.0] + + if all_choices: + return number_list + + probs = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p=probs) + else: + return rng.choice(number_list, p=probs) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Used in Control Mode + new_examples["control_pixel_values"] = [] + + # Used in Inpaint mode + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + length = int(len(pixel_values) // 2) + new_examples["pixel_values"].append(transform(pixel_values)[length:length + 1]) + new_examples["control_pixel_values"].append(transform(control_pixel_values)[length:length + 1]) + + new_examples["text"].append(example["text"]) + + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + + new_examples["mask_pixel_values"].append(mask_pixel_values[:1]) + new_examples["mask"].append(mask[:1]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example for example in new_examples["control_pixel_values"]]) + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + template = args.prompt_template_encode + drop_idx = args.prompt_template_encode_start_idx + + txt = [template.format(e) for e in batch['text']] + txt_tokens = tokenizer( + txt, max_length=args.tokenizer_max_length + drop_idx, padding=True, truncation=True, return_tensors="pt" + ).to(accelerator.device) + encoder_hidden_states = text_encoder( + input_ids=txt_tokens.input_ids, + attention_mask=txt_tokens.attention_mask, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, txt_tokens.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype, device=accelerator.device) + + new_examples['encoder_attention_mask'] = encoder_attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + # Prepare everything with our `accelerator`. + generator_transformer3d.requires_grad_(True) + generator_transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + generator_transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=list(real_score_transformer3d.transformer_blocks) + list(real_score_transformer3d.single_transformer_blocks)) + real_score_transformer3d = shard_fn(real_score_transformer3d) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + real_score_transformer3d.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + args.denoising_step_indices_list = [i for i in range(1000, 0, -25)] + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + denoising_step_list = noise_scheduler.timesteps[args.train_sampling_steps - torch.tensor(args.denoising_step_indices_list)] + + for epoch in range(first_epoch, args.num_train_epochs): + train_cfg_loss = 0.0 + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + mask_pixel_values, mask, texts = batch['mask_pixel_values'].cpu(), batch['mask'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + mask = torch.tile(rearrange(mask, "b f c h w -> b c f h w"), [1, 3, 1, 1, 1]) + for idx, (pixel_value, _mask, text) in enumerate(zip(mask_pixel_values, mask, texts)): + pixel_value = pixel_value[None, ...] + _mask = _mask[None, ...] + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_pixel_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + save_videos_grid(_mask, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = pixel_values.squeeze(1) + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + # Control Latents + control_latents = _batch_encode_vae(control_pixel_values) + control_latents = _patchify_latents(control_latents) + control_latents = ((control_latents - latents_bn_mean) / latents_bn_std).to(dtype=weight_dtype) + control_latents = _pack_latents(control_latents) + + for bs_index in range(control_latents.size()[0]): + if rng is None: + zero_init_control_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + if zero_init_control_conv_in: + control_latents[bs_index] = control_latents[bs_index] * 0 + + mask = rearrange(mask, "b f c h w -> b c f h w").squeeze(2) + mask_conditions = F.interpolate(1 - mask, size=latents.size()[-2:], mode='nearest').to(accelerator.device, weight_dtype) + mask_conditions = _patchify_latents(mask_conditions) + mask_conditions = _pack_latents(mask_conditions) + + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + mask_latents = _patchify_latents(mask_latents) + mask_latents = ((mask_latents - latents_bn_mean) / latents_bn_std).to(dtype=weight_dtype) + mask_latents = _pack_latents(mask_latents) + mask_latents = t2v_flag[:, None, None] * mask_latents + + inpaint_latents = torch.concat([mask_conditions, mask_latents], dim=2) + control_context = torch.cat([control_latents, inpaint_latents], dim=2) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + text_ids = batch['text_ids'] + else: + with torch.no_grad(): + prompt_embeds, text_ids = encode_prompt( + batch['text'], device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + neg_prompt_embeds, neg_text_ids = encode_prompt( + ["低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。构图混乱。文字模糊，扭曲。"], device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + real_score_transformer3d.to(accelerator.device) + torch.cuda.empty_cache() + + bsz, channel, height, width = latents.size() + latents = _patchify_latents(latents) + latent_image_ids = _prepare_latent_ids(latents) + latents = ((latents - latents_bn_mean) / latents_bn_std).to(dtype=weight_dtype) + latents = _pack_latents(latents) + + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + # handle guidance + guidance = torch.tensor([args.guidance_scale], device=accelerator.device) + guidance = guidance.expand(latents.shape[0]) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Student model prediction (single forward pass) + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + student_pred = generator_transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + guidance=guidance, + encoder_hidden_states=prompt_embeds, + txt_ids=text_ids, + img_ids=latent_image_ids, + control_context=control_context, + return_dict=False, + )[0] + + # Teacher model with CFG (two forward passes) + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device), torch.no_grad(): + # Conditional prediction + teacher_pred_cond = real_score_transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + guidance=guidance, + encoder_hidden_states=prompt_embeds, + txt_ids=text_ids, + img_ids=latent_image_ids, + control_context=control_context, + return_dict=False, + )[0] + + # Unconditional prediction + teacher_pred_uncond = real_score_transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + guidance=guidance, + encoder_hidden_states=neg_prompt_embeds, + txt_ids=neg_text_ids, + img_ids=latent_image_ids, + control_context=control_context, + return_dict=False, + )[0] + + # Apply CFG + teacher_pred_cfg = teacher_pred_uncond + ( + teacher_pred_cond - teacher_pred_uncond + ) * args.real_guidance_scale + + # CFG distillation loss + cfg_loss = F.mse_loss( + student_pred.float(), + teacher_pred_cfg.float(), + reduction="mean" + ) + + avg_cfg_loss = accelerator.gather(cfg_loss.repeat(args.train_batch_size)).mean() + train_cfg_loss += avg_cfg_loss.item() / args.gradient_accumulation_steps + + if args.low_vram: + real_score_transformer3d = real_score_transformer3d.to("cpu") + torch.cuda.empty_cache() + + accelerator.backward(cfg_loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_cfg_loss": train_cfg_loss}, step=global_step) + train_cfg_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + + for name, param in generator_transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name not in name: + param.requires_grad = False + break + accelerator.save_state(save_path) + + generator_transformer3d.requires_grad_(True) + logger.info(f"Saved state to {save_path}") + + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + generator_transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"avg_cfg_loss": avg_cfg_loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + generator_transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/flux2_fun/train_control_distill.sh b/scripts/flux2_fun/train_control_distill.sh new file mode 100644 index 0000000..42b9898 --- /dev/null +++ b/scripts/flux2_fun/train_control_distill.sh @@ -0,0 +1,37 @@ +export MODEL_NAME="models/Diffusion_Transformer/FLUX.2-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/flux2_fun/train_control_distill.py \ + --config_path="config/flux2/flux2_control.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_flux2_control_CFG_Distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --low_vram \ + --uniform_sampling \ + --transformer_path="models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union-2602.safetensors" \ + --trainable_modules "control" \ + --random_hw_adapt \ + --resume_from_checkpoint="latest" \ No newline at end of file diff --git a/scripts/hunyuanvideo/train_lora.py b/scripts/hunyuanvideo/train_lora.py index 68617e2..9d5d918 100644 --- a/scripts/hunyuanvideo/train_lora.py +++ b/scripts/hunyuanvideo/train_lora.py @@ -1130,9 +1130,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/longcatvideo/train_lora.py b/scripts/longcatvideo/train_lora.py index 65f31a1..fc109df 100644 --- a/scripts/longcatvideo/train_lora.py +++ b/scripts/longcatvideo/train_lora.py @@ -1039,9 +1039,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/qwenimage/train_edit_lora.py b/scripts/qwenimage/train_edit_lora.py index 54eb289..d54ca3c 100644 --- a/scripts/qwenimage/train_edit_lora.py +++ b/scripts/qwenimage/train_edit_lora.py @@ -966,9 +966,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/qwenimage/train_lora.py b/scripts/qwenimage/train_lora.py index 545a8da..18ba850 100644 --- a/scripts/qwenimage/train_lora.py +++ b/scripts/qwenimage/train_lora.py @@ -922,9 +922,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/wan2.1/train_distill.py b/scripts/wan2.1/train_distill.py index 2123167..8d5c13f 100644 --- a/scripts/wan2.1/train_distill.py +++ b/scripts/wan2.1/train_distill.py @@ -945,6 +945,8 @@ def main(): state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict m, u = generator_transformer3d.load_state_dict(state_dict, strict=False) + m, u = real_score_transformer3d.load_state_dict(state_dict, strict=False) + m, u = fake_score_transformer3d.load_state_dict(state_dict, strict=False) print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") assert len(u) == 0 diff --git a/scripts/wan2.1/train_distill_lora.py b/scripts/wan2.1/train_distill_lora.py index e435514..a62d69e 100644 --- a/scripts/wan2.1/train_distill_lora.py +++ b/scripts/wan2.1/train_distill_lora.py @@ -1007,6 +1007,8 @@ def main(): state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict m, u = generator_transformer3d.load_state_dict(state_dict, strict=False) + m, u = real_score_transformer3d.load_state_dict(state_dict, strict=False) + m, u = fake_score_transformer3d.load_state_dict(state_dict, strict=False) print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") assert len(u) == 0 @@ -1122,9 +1124,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - accelerator_fake_score_transformer3d.register_save_state_pre_hook(save_model_hook) accelerator_fake_score_transformer3d.register_load_state_pre_hook(load_model_hook) diff --git a/scripts/wan2.1/train_lora.py b/scripts/wan2.1/train_lora.py index 2319f77..4c796e6 100755 --- a/scripts/wan2.1/train_lora.py +++ b/scripts/wan2.1/train_lora.py @@ -1059,9 +1059,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/wan2.1_fun/train_control_lora.py b/scripts/wan2.1_fun/train_control_lora.py index ff23604..26e0d42 100755 --- a/scripts/wan2.1_fun/train_control_lora.py +++ b/scripts/wan2.1_fun/train_control_lora.py @@ -970,9 +970,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/wan2.1_fun/train_lora.py b/scripts/wan2.1_fun/train_lora.py index 55bab95..9fbedd4 100755 --- a/scripts/wan2.1_fun/train_lora.py +++ b/scripts/wan2.1_fun/train_lora.py @@ -1029,9 +1029,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/wan2.2/train_animate_lora.py b/scripts/wan2.2/train_animate_lora.py index d874ad1..29b5c96 100644 --- a/scripts/wan2.2/train_animate_lora.py +++ b/scripts/wan2.2/train_animate_lora.py @@ -1045,9 +1045,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/wan2.2/train_distill_lora.py b/scripts/wan2.2/train_distill_lora.py index fb331b9..812b009 100644 --- a/scripts/wan2.2/train_distill_lora.py +++ b/scripts/wan2.2/train_distill_lora.py @@ -1169,9 +1169,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - accelerator_fake_score_transformer3d.register_save_state_pre_hook(save_model_hook) accelerator_fake_score_transformer3d.register_load_state_pre_hook(load_model_hook) diff --git a/scripts/wan2.2/train_lora.py b/scripts/wan2.2/train_lora.py index b1751d0..0bb8778 100755 --- a/scripts/wan2.2/train_lora.py +++ b/scripts/wan2.2/train_lora.py @@ -1111,9 +1111,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/wan2.2/train_s2v_lora.py b/scripts/wan2.2/train_s2v_lora.py index 68b6a59..4f0af45 100644 --- a/scripts/wan2.2/train_s2v_lora.py +++ b/scripts/wan2.2/train_s2v_lora.py @@ -1095,9 +1095,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/wan2.2_fun/train_control_lora.py b/scripts/wan2.2_fun/train_control_lora.py index 1823d5e..4f9018e 100644 --- a/scripts/wan2.2_fun/train_control_lora.py +++ b/scripts/wan2.2_fun/train_control_lora.py @@ -1059,9 +1059,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/wan2.2_fun/train_lora.py b/scripts/wan2.2_fun/train_lora.py index 7cb2105..be42be3 100644 --- a/scripts/wan2.2_fun/train_lora.py +++ b/scripts/wan2.2_fun/train_lora.py @@ -1074,9 +1074,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/z_image/train_lora.py b/scripts/z_image/train_lora.py index d74e509..7d92de8 100644 --- a/scripts/z_image/train_lora.py +++ b/scripts/z_image/train_lora.py @@ -946,9 +946,6 @@ def main(): accelerator.register_save_state_pre_hook(save_model_hook) accelerator.register_load_state_pre_hook(load_model_hook) - accelerator.register_save_state_pre_hook(save_model_hook) - accelerator.register_load_state_pre_hook(load_model_hook) - if args.gradient_checkpointing: transformer3d.enable_gradient_checkpointing() diff --git a/scripts/z_image_fun/train_control_2.1.sh b/scripts/z_image_fun/train_control_2.1.sh index 586b0d8..8473a62 100644 --- a/scripts/z_image_fun/train_control_2.1.sh +++ b/scripts/z_image_fun/train_control_2.1.sh @@ -1,4 +1,4 @@ -export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export MODEL_NAME="models/Diffusion_Transformer/Z-Image" export DATASET_NAME="datasets/internal_datasets/" export DATASET_META_NAME="datasets/internal_datasets/metadata.json" # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. @@ -31,5 +31,5 @@ accelerate launch --mixed_precision="bf16" scripts/z_image_fun/train_control.py --enable_bucket \ --uniform_sampling \ --add_inpaint_info \ - --transformer_path="models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1.safetensors" \ + --transformer_path="models/Personalized_Model/Z-Image-Fun-Controlnet-Union-2.1.safetensors" \ --trainable_modules "control" \ No newline at end of file diff --git a/scripts/z_image_fun/train_control_distill.py b/scripts/z_image_fun/train_control_distill.py index 27ddb3a..dc91a51 100644 --- a/scripts/z_image_fun/train_control_distill.py +++ b/scripts/z_image_fun/train_control_distill.py @@ -1659,6 +1659,8 @@ def main(): if args.low_vram and not args.enable_text_encoder_in_dataloader: text_encoder.to('cpu') torch.cuda.empty_cache() + if args.low_vram: + real_score_transformer3d = real_score_transformer3d.to(accelerator.device) with accelerator.accumulate(generator_transformer3d): def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): diff --git a/scripts/z_image_fun/train_control.sh b/scripts/z_image_fun/train_turbo_control.sh similarity index 100% rename from scripts/z_image_fun/train_control.sh rename to scripts/z_image_fun/train_turbo_control.sh diff --git a/scripts/z_image_fun/train_control_2.0.sh b/scripts/z_image_fun/train_turbo_control_2.0.sh similarity index 100% rename from scripts/z_image_fun/train_control_2.0.sh rename to scripts/z_image_fun/train_turbo_control_2.0.sh diff --git a/scripts/z_image_fun/train_turbo_control_2.1.sh b/scripts/z_image_fun/train_turbo_control_2.1.sh new file mode 100644 index 0000000..586b0d8 --- /dev/null +++ b/scripts/z_image_fun/train_turbo_control_2.1.sh @@ -0,0 +1,35 @@ +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/z_image_fun/train_control.py \ + --config_path="config/z_image/z_image_control_2.1.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_z_image_control" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --add_inpaint_info \ + --transformer_path="models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union-2.1.safetensors" \ + --trainable_modules "control" \ No newline at end of file diff --git a/scripts/z_image_fun/train_control_distill.sh b/scripts/z_image_fun/train_turbo_control_distill.sh similarity index 100% rename from scripts/z_image_fun/train_control_distill.sh rename to scripts/z_image_fun/train_turbo_control_distill.sh diff --git a/videox_fun/models/flux2_transformer2d_control.py b/videox_fun/models/flux2_transformer2d_control.py index 100abbb..1a11910 100644 --- a/videox_fun/models/flux2_transformer2d_control.py +++ b/videox_fun/models/flux2_transformer2d_control.py @@ -28,6 +28,7 @@ from .flux2_transformer2d import (Flux2SingleTransformerBlock, Flux2Transformer2DModel, Flux2TransformerBlock) +VIDEOX_OFFLOAD_VACE_LATENTS = os.environ.get("VIDEOX_OFFLOAD_VACE_LATENTS", False) class Flux2ControlTransformerBlock(Flux2TransformerBlock): def __init__( @@ -58,8 +59,16 @@ class Flux2ControlTransformerBlock(Flux2TransformerBlock): all_c = list(torch.unbind(c)) c = all_c.pop(-1) + if VIDEOX_OFFLOAD_VACE_LATENTS: + c = c.to(x.device) + encoder_hidden_states, c = super().forward(c, **kwargs) c_skip = self.after_proj(c) + + if VIDEOX_OFFLOAD_VACE_LATENTS: + c_skip = c_skip.to("cpu") + c = c.to("cpu") + all_c += [c_skip, c] c = torch.stack(all_c) return encoder_hidden_states, c @@ -82,7 +91,11 @@ class BaseFlux2TransformerBlock(Flux2TransformerBlock): def forward(self, hidden_states, hints=None, context_scale=1.0, **kwargs): encoder_hidden_states, hidden_states = super().forward(hidden_states, **kwargs) if self.block_id is not None: - hidden_states = hidden_states + hints[self.block_id] * context_scale + if VIDEOX_OFFLOAD_VACE_LATENTS: + hidden_states = hidden_states + hints[self.block_id].to(hidden_states.device) * context_scale + else: + hidden_states = hidden_states + hints[self.block_id] * context_scale + return encoder_hidden_states, hidden_states diff --git a/videox_fun/models/z_image_transformer2d_control.py b/videox_fun/models/z_image_transformer2d_control.py index 294d146..9aed252 100644 --- a/videox_fun/models/z_image_transformer2d_control.py +++ b/videox_fun/models/z_image_transformer2d_control.py @@ -40,7 +40,7 @@ from .z_image_transformer2d import (ZImageTransformer2DModel, FinalLayer, ADALN_EMBED_DIM = 256 SEQ_MULTI_OF = 32 - +VIDEOX_OFFLOAD_VACE_LATENTS = os.environ.get("VIDEOX_OFFLOAD_VACE_LATENTS", False) class ZImageControlTransformerBlock(ZImageTransformerBlock): def __init__( @@ -72,8 +72,16 @@ class ZImageControlTransformerBlock(ZImageTransformerBlock): all_c = list(torch.unbind(c)) c = all_c.pop(-1) + if VIDEOX_OFFLOAD_VACE_LATENTS: + c = c.to(x.device) + c = super().forward(c, **kwargs) c_skip = self.after_proj(c) + + if VIDEOX_OFFLOAD_VACE_LATENTS: + c_skip = c_skip.to("cpu") + c = c.to("cpu") + all_c += [c_skip, c] c = torch.stack(all_c) return c @@ -97,8 +105,12 @@ class BaseZImageTransformerBlock(ZImageTransformerBlock): def forward(self, hidden_states, hints=None, context_scale=1.0, **kwargs): hidden_states = super().forward(hidden_states, **kwargs) if self.block_id is not None: - hidden_states = hidden_states + hints[self.block_id] * context_scale + if VIDEOX_OFFLOAD_VACE_LATENTS: + hidden_states = hidden_states + hints[self.block_id].to(hidden_states.device) * context_scale + else: + hidden_states = hidden_states + hints[self.block_id] * context_scale return hidden_states + class ZImageControlTransformer2DModel(ZImageTransformer2DModel): @register_to_config diff --git a/videox_fun/pipeline/pipeline_flux2.py b/videox_fun/pipeline/pipeline_flux2.py index 06488f7..5a9d263 100644 --- a/videox_fun/pipeline/pipeline_flux2.py +++ b/videox_fun/pipeline/pipeline_flux2.py @@ -621,11 +621,13 @@ class Flux2Pipeline(DiffusionPipeline): self, image: Optional[Union[List[PIL.Image.Image], PIL.Image.Image]] = None, prompt: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, height: Optional[int] = None, width: Optional[int] = None, num_inference_steps: int = 50, sigmas: Optional[List[float]] = None, guidance_scale: Optional[float] = 4.0, + true_cfg_scale: Optional[float] = 1.0, num_images_per_prompt: int = 1, generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, latents: Optional[torch.Tensor] = None, @@ -734,6 +736,8 @@ class Flux2Pipeline(DiffusionPipeline): batch_size = prompt_embeds.shape[0] device = self._execution_device + has_neg_prompt = negative_prompt is not None + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt if comfyui_progressbar: from comfy.utils import ProgressBar pbar = ProgressBar(num_inference_steps + 2) @@ -747,6 +751,15 @@ class Flux2Pipeline(DiffusionPipeline): max_sequence_length=max_sequence_length, text_encoder_out_layers=text_encoder_out_layers, ) + if do_true_cfg: + negative_prompt_embeds, negative_text_ids = self.encode_prompt( + prompt=negative_prompt, + prompt_embeds=None, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + text_encoder_out_layers=text_encoder_out_layers, + ) # 4. process images if image is not None and not isinstance(image, list): @@ -854,9 +867,22 @@ class Flux2Pipeline(DiffusionPipeline): joint_attention_kwargs=self._attention_kwargs, return_dict=False, )[0] - noise_pred = noise_pred[:, : latents.size(1) :] + if do_true_cfg: + neg_noise_pred = self.transformer( + hidden_states=latent_model_input, # (B, image_seq_len, C) + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states=negative_prompt_embeds, + txt_ids=negative_text_ids, # B, text_seq_len, 4 + img_ids=latent_image_ids, # B, image_seq_len, 4 + joint_attention_kwargs=self._attention_kwargs, + return_dict=False, + )[0] + neg_noise_pred = neg_noise_pred[:, : latents.size(1) :] + noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + # compute the previous noisy sample x_t -> x_t-1 latents_dtype = latents.dtype latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] diff --git a/videox_fun/pipeline/pipeline_flux2_control.py b/videox_fun/pipeline/pipeline_flux2_control.py index c3b1613..ea8223a 100644 --- a/videox_fun/pipeline/pipeline_flux2_control.py +++ b/videox_fun/pipeline/pipeline_flux2_control.py @@ -626,6 +626,7 @@ class Flux2ControlPipeline(DiffusionPipeline): def __call__( self, prompt: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, height: Optional[int] = None, width: Optional[int] = None, @@ -638,6 +639,7 @@ class Flux2ControlPipeline(DiffusionPipeline): num_inference_steps: int = 50, sigmas: Optional[List[float]] = None, guidance_scale: Optional[float] = 4.0, + true_cfg_scale: Optional[float] = 1.0, num_images_per_prompt: int = 1, generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, latents: Optional[torch.Tensor] = None, @@ -758,6 +760,9 @@ class Flux2ControlPipeline(DiffusionPipeline): height = height or self.default_sample_size * self.vae_scale_factor width = width or self.default_sample_size * self.vae_scale_factor num_channels_latents = self.transformer.config.in_channels // 4 + + has_neg_prompt = negative_prompt is not None + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt # Prepare mask latent variables if mask_image is not None: @@ -813,6 +818,15 @@ class Flux2ControlPipeline(DiffusionPipeline): max_sequence_length=max_sequence_length, text_encoder_out_layers=text_encoder_out_layers, ) + if do_true_cfg: + negative_prompt_embeds, negative_text_ids = self.encode_prompt( + prompt=negative_prompt, + prompt_embeds=None, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + text_encoder_out_layers=text_encoder_out_layers, + ) # 4. process images if image is not None and not isinstance(image, list): @@ -933,9 +947,24 @@ class Flux2ControlPipeline(DiffusionPipeline): control_context_scale=control_context_scale, return_dict=False, )[0] - noise_pred = noise_pred[:, : latents.size(1) :] + if do_true_cfg: + neg_noise_pred = self.transformer( + hidden_states=latent_model_input, # (B, image_seq_len, C) + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states=negative_prompt_embeds, + txt_ids=negative_text_ids, # B, text_seq_len, 4 + img_ids=latent_image_ids, # B, image_seq_len, 4 + joint_attention_kwargs=self._attention_kwargs, + control_context=control_context_input, + control_context_scale=control_context_scale, + return_dict=False, + )[0] + neg_noise_pred = neg_noise_pred[:, : latents.size(1) :] + noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + # compute the previous noisy sample x_t -> x_t-1 latents_dtype = latents.dtype latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]