Mochi Inferenfce & Diffusers

README.md

@@ -8,8 +8,8 @@ conda create -n fastvideo python=3.10.12
 conda activate fastvideo
 pip3 install torch==2.5.0 torchvision  --index-url https://download.pytorch.org/whl/cu121
 pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu121
-pip install ninja
-pip install flash-attn --no-build-isolationf
+cd .. && git clone  https://github.com/huggingface/diffusers
+cd diffusers && git checkout mochi && pip install -e . && cd ../fastvideo
 ```
 
 ```
@@ -22,10 +22,10 @@ We've prepared some debug data to facilitate development. To make sure the train
 ```
 python scripts/download_hf.py --repo_id=Stealths-Video/dummyVid --local_dir=data/dummyVid --repo_type=dataset
 python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0  --file_name vae/checkpoint.ckpt --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model 
+python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0  --file_name vae/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model 
 python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0  --file_name 29x480p/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model
 python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0  --file_name 29x480p/diffusion_pytorch_model.safetensors --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model
-python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0  --file_name vae/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model 
-python scripts/download_hf.py --repo_id=genmo/mochi-1-preview  --local_dir=data/mochi-1-preview --repo_type=model
+python scripts/download_hf.py --repo_id=Stealths-Video/mochi --local_dir=data/mochi --repo_type=model
 ```
 
 ```

opensora/sample/sample_t2v_mochi.py

@@ -0,0 +1,37 @@
+import torch
+from diffusers import MochiPipeline
+
+from diffusers.utils import export_to_video, load_image, load_video
+
+def main():
+    # Set the random seed for reproducibility
+    generator = torch.Generator("cpu").manual_seed(12345)
+
+    # Load the Mochi pipeline
+    model_path = "data/mochi"
+    prompt = "A hand with delicate fingers picks up a bright yellow lemon from a wooden bowl filled with lemons and sprigs of mint against a peach-colored background. The hand gently tosses the lemon up and catches it, showcasing its smooth texture. A beige string bag sits beside the bowl, adding a rustic touch to the scene. Additional lemons, one halved, are scattered around the base of the bowl. The even lighting enhances the vibrant colors and creates a fresh, inviting atmosphere."
+    num_frames = 163
+    height = 480
+    width = 848
+    num_inference_steps = 64
+
+    pipe = MochiPipeline.from_pretrained(model_path, torch_dtype=torch.float16)
+    pipe.enable_vae_tiling()
+    pipe.enable_model_cpu_offload()
+
+    # Generate videos from the input prompt
+    video = pipe(
+        prompt=prompt,
+        height=height,
+        width=width,
+        num_frames=num_frames,
+        generator=generator,
+        num_inference_steps=num_inference_steps,
+        guidance_scale=4.5
+    ).frames[0]
+
+
+    export_to_video(video, './outputs.mp4', fps=30)
+
+if __name__ == "__main__":
+    main()

opensora/train/train_t2v_diffusers.py

@@ -45,9 +45,8 @@ from diffusers.utils import check_min_version, is_wandb_available
 from opensora.utils.ema import EMAModel
 from opensora.dataset import getdataset
 from opensora.models import CausalVAEModelWrapper
-from opensora.models.text_encoder import get_text_enc, get_text_warpper
+from opensora.models.text_encoder import  get_text_warpper
 from opensora.models.causalvideovae import ae_stride_config, ae_channel_config
-from opensora.models.causalvideovae import ae_norm, ae_denorm
 from opensora.models.diffusion import Diffusion_models, Diffusion_models_class
 from opensora.utils.dataset_utils import Collate, LengthGroupedSampler
 from opensora.sample.pipeline_opensora import OpenSoraPipeline