diff --git a/README.md b/README.md index 292b6e0d4..d48c6da61 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ conda create -n fastvideo python=3.10.12 conda activate fastvideo pip3 install torch==2.5.0 torchvision --index-url https://download.pytorch.org/whl/cu121 pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu121 -pip install ninja -pip install flash-attn --no-build-isolationf +cd .. && git clone https://github.com/huggingface/diffusers +cd diffusers && git checkout mochi && pip install -e . && cd ../fastvideo ``` ``` @@ -22,10 +22,10 @@ We've prepared some debug data to facilitate development. To make sure the train ``` python scripts/download_hf.py --repo_id=Stealths-Video/dummyVid --local_dir=data/dummyVid --repo_type=dataset python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name vae/checkpoint.ckpt --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model +python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name vae/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name 29x480p/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name 29x480p/diffusion_pytorch_model.safetensors --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model -python scripts/download_hf.py --repo_id=LanguageBind/Open-Sora-Plan-v1.2.0 --file_name vae/config.json --local_dir=data/Open-Sora-Plan-v1.2.0 --repo_type=model -python scripts/download_hf.py --repo_id=genmo/mochi-1-preview --local_dir=data/mochi-1-preview --repo_type=model +python scripts/download_hf.py --repo_id=Stealths-Video/mochi --local_dir=data/mochi --repo_type=model ``` ``` diff --git a/opensora/sample/sample_t2v_mochi.py b/opensora/sample/sample_t2v_mochi.py new file mode 100644 index 000000000..a47d38b1f --- /dev/null +++ b/opensora/sample/sample_t2v_mochi.py @@ -0,0 +1,37 @@ +import torch +from diffusers import MochiPipeline + +from diffusers.utils import export_to_video, load_image, load_video + +def main(): + # Set the random seed for reproducibility + generator = torch.Generator("cpu").manual_seed(12345) + + # Load the Mochi pipeline + model_path = "data/mochi" + prompt = "A hand with delicate fingers picks up a bright yellow lemon from a wooden bowl filled with lemons and sprigs of mint against a peach-colored background. The hand gently tosses the lemon up and catches it, showcasing its smooth texture. A beige string bag sits beside the bowl, adding a rustic touch to the scene. Additional lemons, one halved, are scattered around the base of the bowl. The even lighting enhances the vibrant colors and creates a fresh, inviting atmosphere." + num_frames = 163 + height = 480 + width = 848 + num_inference_steps = 64 + + pipe = MochiPipeline.from_pretrained(model_path, torch_dtype=torch.float16) + pipe.enable_vae_tiling() + pipe.enable_model_cpu_offload() + + # Generate videos from the input prompt + video = pipe( + prompt=prompt, + height=height, + width=width, + num_frames=num_frames, + generator=generator, + num_inference_steps=num_inference_steps, + guidance_scale=4.5 + ).frames[0] + + + export_to_video(video, './outputs.mp4', fps=30) + +if __name__ == "__main__": + main() diff --git a/opensora/train/train_t2v_diffusers.py b/opensora/train/train_t2v_diffusers.py index b6861cfff..967642b25 100644 --- a/opensora/train/train_t2v_diffusers.py +++ b/opensora/train/train_t2v_diffusers.py @@ -45,9 +45,8 @@ from diffusers.utils import check_min_version, is_wandb_available from opensora.utils.ema import EMAModel from opensora.dataset import getdataset from opensora.models import CausalVAEModelWrapper -from opensora.models.text_encoder import get_text_enc, get_text_warpper +from opensora.models.text_encoder import get_text_warpper from opensora.models.causalvideovae import ae_stride_config, ae_channel_config -from opensora.models.causalvideovae import ae_norm, ae_denorm from opensora.models.diffusion import Diffusion_models, Diffusion_models_class from opensora.utils.dataset_utils import Collate, LengthGroupedSampler from opensora.sample.pipeline_opensora import OpenSoraPipeline