'update'

README.md

@@ -6,6 +6,17 @@
  
 Based on https://github.com/kijai/ComfyUI-CogVideoXWrapper add model loader and some other features, you need to install ComfyUI-CogVideoXWrapper before using
 
+## Update - 2024-09-22
+支持CogVideoX-I2V图生视频, 提供Q4量化模型
+
+(support CogVideoX-I2V image to video, provide Q4 quantization model)
+
+[CogVideoX_5b_I2V_GGUF_Q4_0.safetensors](https://huggingface.co/Kijai/CogVideoX_GGUF/resolve/main/CogVideoX_5b_I2V_GGUF_Q4_0.safetensors) 下载到unet文件夹 (Donwload to unet folder)
+
+工作流在examples\workflow_I2V_GGUF_Q4_0.png (Workflow in examples\workflow_I2V_GGUF_Q4_0.png)
+
+Prompt :The girl in the video happily puts on sunglasses.
+
 
 ## Update - 2024-09-19
 支持CogVideoX-Fun图生视频, 提供Q4量化模型
@@ -14,7 +25,7 @@ Based on https://github.com/kijai/ComfyUI-CogVideoXWrapper add model loader and
 
 [CogVideoX_Fun_GGUF_Q4_0.safetensors](https://modelscope.cn/models/wailovet/CogVideoX-5b/resolve/master/CogVideoX_5b_fun_GGUF_Q4_0.safetensors) 下载到unet文件夹 (Donwload to unet folder)
 
-工作流在examples\workflow_I2V_GGUF_Q4_0.png (Workflow in examples\workflow_I2V_GGUF_Q4_0.png)
+工作流在examples\workflow_FUN_I2V_GGUF_Q4_0.png (Workflow in examples\workflow_FUN_I2V_GGUF_Q4_0.png)
 
 Prompt :The girl in the video happily puts on sunglasses.
 

configs5b-i2v/scheduler/scheduler_config.json

@@ -0,0 +1,18 @@
+{
+  "_class_name": "CogVideoXDDIMScheduler",
+  "_diffusers_version": "0.31.0.dev0",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "clip_sample_range": 1.0,
+  "num_train_timesteps": 1000,
+  "prediction_type": "v_prediction",
+  "rescale_betas_zero_snr": true,
+  "sample_max_value": 1.0,
+  "set_alpha_to_one": true,
+  "snr_shift_scale": 1.0,
+  "steps_offset": 0,
+  "timestep_spacing": "trailing",
+  "trained_betas": null
+}

mz_cogvideox_core.py

@@ -181,7 +181,7 @@ cogVideoXDDIMSchedulerConfig5B = {
     "set_alpha_to_one": True,
     "snr_shift_scale": 1.0,
     "steps_offset": 0,
-    "timestep_spacing": "linspace",
+    "timestep_spacing": "trailing",
     "trained_betas": None,
 }
 
@@ -266,6 +266,8 @@ def MZ_CogVideoXLoader_call(args={}):
         transformer_type = "fun_2b"
     elif unet_sd["patch_embed.proj.weight"].shape == (1920, 16, 2, 2):
         transformer_type = "2b"
+    elif unet_sd["patch_embed.proj.weight"].shape == (3072, 32, 2, 2):
+        transformer_type = "i2v_5b"
     else:
         raise Exception("This model is not supported")
 
@@ -296,6 +298,13 @@ def MZ_CogVideoXLoader_call(args={}):
                 os.path.dirname(__file__),
                 "configs5b-Fun",
             )
+        elif transformer_type == "i2v_5b":
+            transformer_config["in_channels"] = 32
+            transformer_config["use_learned_positional_embeddings"] = True
+            base_path = os.path.join(
+                os.path.dirname(__file__),
+                "configs5b-i2v",
+            )
 
     if transformer_type.endswith("2b"):
         transformer_config = cogVideoXTransformerConfig
@@ -362,9 +371,14 @@ def MZ_CogVideoXLoader_call(args={}):
             print("convert to fp8 linear")
             convert_fp8_linear(transformer, weight_dtype, manual_cast_dtype)
 
-        if transformer_type.endswith("2b"):
-            transformer.pos_embedding = transformer.pos_embedding.to(
-                manual_cast_dtype)
+        if transformer_type.endswith("2b") or transformer_type == "i2v_5b":
+            if hasattr(transformer, "pos_embedding"):
+                transformer.pos_embedding = transformer.pos_embedding.to(
+                    manual_cast_dtype)
+            if hasattr(transformer, "patch_embed") and hasattr(transformer.patch_embed, "pos_embedding"):
+                transformer.patch_embed.pos_embedding = transformer.patch_embed.pos_embedding.to(
+                    manual_cast_dtype)
+             
 
     transformer.to(device)