diff --git a/README.md b/README.md index 498c5b8..94fcf9b 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,17 @@ Based on https://github.com/kijai/ComfyUI-CogVideoXWrapper add model loader and some other features, you need to install ComfyUI-CogVideoXWrapper before using +## Update - 2024-09-22 +支持CogVideoX-I2V图生视频, 提供Q4量化模型 + +(support CogVideoX-I2V image to video, provide Q4 quantization model) + +[CogVideoX_5b_I2V_GGUF_Q4_0.safetensors](https://huggingface.co/Kijai/CogVideoX_GGUF/resolve/main/CogVideoX_5b_I2V_GGUF_Q4_0.safetensors) 下载到unet文件夹 (Donwload to unet folder) + +工作流在examples\workflow_I2V_GGUF_Q4_0.png (Workflow in examples\workflow_I2V_GGUF_Q4_0.png) + +Prompt :The girl in the video happily puts on sunglasses. + ## Update - 2024-09-19 支持CogVideoX-Fun图生视频, 提供Q4量化模型 @@ -14,7 +25,7 @@ Based on https://github.com/kijai/ComfyUI-CogVideoXWrapper add model loader and [CogVideoX_Fun_GGUF_Q4_0.safetensors](https://modelscope.cn/models/wailovet/CogVideoX-5b/resolve/master/CogVideoX_5b_fun_GGUF_Q4_0.safetensors) 下载到unet文件夹 (Donwload to unet folder) -工作流在examples\workflow_I2V_GGUF_Q4_0.png (Workflow in examples\workflow_I2V_GGUF_Q4_0.png) +工作流在examples\workflow_FUN_I2V_GGUF_Q4_0.png (Workflow in examples\workflow_FUN_I2V_GGUF_Q4_0.png) Prompt :The girl in the video happily puts on sunglasses. diff --git a/configs5b-i2v/scheduler/scheduler_config.json b/configs5b-i2v/scheduler/scheduler_config.json new file mode 100644 index 0000000..6e4f799 --- /dev/null +++ b/configs5b-i2v/scheduler/scheduler_config.json @@ -0,0 +1,18 @@ +{ + "_class_name": "CogVideoXDDIMScheduler", + "_diffusers_version": "0.31.0.dev0", + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "beta_start": 0.00085, + "clip_sample": false, + "clip_sample_range": 1.0, + "num_train_timesteps": 1000, + "prediction_type": "v_prediction", + "rescale_betas_zero_snr": true, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "snr_shift_scale": 1.0, + "steps_offset": 0, + "timestep_spacing": "trailing", + "trained_betas": null +} \ No newline at end of file diff --git a/examples/workflow_FUN_I2V_GGUF_Q4_0.png b/examples/workflow_FUN_I2V_GGUF_Q4_0.png new file mode 100644 index 0000000..2c32e0f Binary files /dev/null and b/examples/workflow_FUN_I2V_GGUF_Q4_0.png differ diff --git a/examples/workflow_I2V_GGUF_Q4_0.png b/examples/workflow_I2V_GGUF_Q4_0.png index 2c32e0f..a98de8a 100644 Binary files a/examples/workflow_I2V_GGUF_Q4_0.png and b/examples/workflow_I2V_GGUF_Q4_0.png differ diff --git a/mz_cogvideox_core.py b/mz_cogvideox_core.py index a05741d..a79dee5 100644 --- a/mz_cogvideox_core.py +++ b/mz_cogvideox_core.py @@ -181,7 +181,7 @@ cogVideoXDDIMSchedulerConfig5B = { "set_alpha_to_one": True, "snr_shift_scale": 1.0, "steps_offset": 0, - "timestep_spacing": "linspace", + "timestep_spacing": "trailing", "trained_betas": None, } @@ -266,6 +266,8 @@ def MZ_CogVideoXLoader_call(args={}): transformer_type = "fun_2b" elif unet_sd["patch_embed.proj.weight"].shape == (1920, 16, 2, 2): transformer_type = "2b" + elif unet_sd["patch_embed.proj.weight"].shape == (3072, 32, 2, 2): + transformer_type = "i2v_5b" else: raise Exception("This model is not supported") @@ -296,6 +298,13 @@ def MZ_CogVideoXLoader_call(args={}): os.path.dirname(__file__), "configs5b-Fun", ) + elif transformer_type == "i2v_5b": + transformer_config["in_channels"] = 32 + transformer_config["use_learned_positional_embeddings"] = True + base_path = os.path.join( + os.path.dirname(__file__), + "configs5b-i2v", + ) if transformer_type.endswith("2b"): transformer_config = cogVideoXTransformerConfig @@ -362,9 +371,14 @@ def MZ_CogVideoXLoader_call(args={}): print("convert to fp8 linear") convert_fp8_linear(transformer, weight_dtype, manual_cast_dtype) - if transformer_type.endswith("2b"): - transformer.pos_embedding = transformer.pos_embedding.to( - manual_cast_dtype) + if transformer_type.endswith("2b") or transformer_type == "i2v_5b": + if hasattr(transformer, "pos_embedding"): + transformer.pos_embedding = transformer.pos_embedding.to( + manual_cast_dtype) + if hasattr(transformer, "patch_embed") and hasattr(transformer.patch_embed, "pos_embedding"): + transformer.patch_embed.pos_embedding = transformer.patch_embed.pos_embedding.to( + manual_cast_dtype) + transformer.to(device)