prepare scripts

.gitignore

@@ -22,7 +22,8 @@ wandb/
 cache_dir/
 wandb/
 test*
-sample_videos*
+sample_video*
+sample_image*
 512*
 720*
 1024*
@@ -40,4 +41,5 @@ YSH*
 hw*
 257f*
 513f*
-taming*
+taming*
+221hw*

examples/prompt_list_1.txt

@@ -1,7 +0,0 @@
-A calico cat wearing aviator sunglasses and a leather jacket, sitting confidently on a motorcycle.
-A penguin dressed in suits and bow ties, conducting a symphony orchestra on a glittering ice floe.
-An abandoned amusement park at dusk, with a carousel slowly turning and ghostly laughter echoing in the air.
-A pirate ship caught in a stormy sea, battling high waves as lightning illuminates the fierce crew preparing for battle.
-The video showcases a typical urban street scene with multi-story residential buildings in the background and several vehicles, including a blue truck and a red car, parked on the street. Initially, a person is seen standing next to the blue truck, engaging in some activity, while another individual walks past the parked cars. As the video progresses, the scene remains consistent with the urban setting and the positions of the vehicles unchanged. However, the actions of the individuals evolve slightly; one person continues to walk past the parked cars, while the other individual's activity near the blue truck persists. The overall environment and layout of the street remain consi stent throughout the video, with the primary change being the subtle movements and actions of the people present.
-The video showcases a close-up view of a yellow kitchen appliance or equipment in the background, with the focus on a crispy, golden-brown meat, likely a chicken leg or drumstick, seasoned or coated with spices or batter, in the foreground. Accompanying the meat are purple onion slices or pieces and a lime wedge or slice. As the video progresses, the texture of the meat becomes more de tailed, revealing visible pores or imperfections on its skin. The arrangement of the onion pieces isslightly altered, indicating some movement or rearrangement. Throughout the video, the composition maintains its focus on the meat and onion pieces against the slightly blurred background of the yellow kitchen object.
-The video showcases a serene and tranquil body of water, characterized by its vibrant green hue that reflects the surrounding foliage. The surface of the water is gently rippled, with delicate white flower petals floating on it, contributing to the scene's peaceful ambiance. The reflections on the water's surface create a mesmerizing interplay of light and color. In the background, a rocky shoreline is visible, providing a natural contrast to the vivid hues of the water. Throughout the video, the overall atmosphere remains one of peace and tranquility, inviting viewers to appreciate the beauty of nature's simplicity. The scene remains largely unchanged, maintaining its serene and tranquil character from beginning to end.

examples/sora.txt

@@ -1,51 +0,0 @@
-A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage. She wears a black leather jacket, along red dress, and black boots, and carries a black purse. She wears sunglasses and red lipstick. She walks confidently and casually. The street is dampand reflective, creating a mirror effect of thecolorful lights. Many pedestrians walk about.
-Several giant wooly mammoths approach treading through a snowy meadow, their long wooly fur lightly blows in the wind as they walk, snow covered tree sand dramatic snow capped mountains in the distance,mid afternoon lightwith wispy cloud sand a sun high in the distance creates a warm glow, the low camera view is stunning capturing the large furry mammal with beautiful photography, depth of field
-A movie trailer featuring the adventures ofthe 30 year old spacemanwearing a redwool knitted motorcycle helmet, bluesky, saltdesert, cinematic style, shoton 35mm film, vivid colors. 
-Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach.The crashing blue waters create white-tipped waves,while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green
-shrubbery covers the cliffs edge. The steep drop from the road down to the beach is adramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.
-Animated scene features a close-up of a short fluffy monster kneeling beside a melting red candle.The art style is 3D and realistic,with a focus on lighting and texture.The mood of the painting is one of wonder and curiosity, as the monster gazes at the flame with wide eyes and
-open mouth. lts pose and expression convey a sense of innocence and playfulness, as if it is exploring the world around it for the first time.The use of warm colors and dramatic lighting further enhances the cozy atmosphere of the image.
-A gorgeously rendered papercraft world of a coral reef,rife with colorful fish and sea creatures.
-This close-up shot of a Victoria crowned pigeon showcases its striking blue plumage and red chest. Its crest is made of delicate, lacy feathers, while its eye is a striking red color. The bird's head is tilted slightly to the side,giving the impression of it looking regal and majestic. The background is blurred,drawing attention to the bird's striking appearance.
-Photorealistic closeup video of two pirate ships battling each other as they sail inside a cup of coffee.
-A young man at his 20s is sitting on a piece of cloud in the sky, reading a book.
-A petri dish with a bamboo forest growing within it that has tiny red pandas running around.
-The camera rotates around a large stack of vintage televisions all showing different programs-1950s sci-fi movies, horror movies, news, static, a 1970s sitcom, etc, set inside a large New York museum gallery.
-3D animation of a small, round, fluffy creature with big, expressive eyes explores a vibrant, enchanted forest. The creature, a whimsical blend of a rabbit and a squirrel, has soft blue fur and a bushy, striped tail. It hops along a sparkling stream,its eyes wide with wonder. The forest is alive with magical elements: flowers that glow and change colors, trees with leaves in shades of purple and silver, and small floating lights that resemble fireflies. The creature stops to interact playfully with a group of tiny, fairy-like beings dancing around a mushroom ring. The creature looks up in awe at a large, glowing tree that seems to be the heart of the forest.
-Historical footage of California during the gold rush.
-A close up view of a glass sphere that has a zen garden within it. There is a small dwarf in the sphere who is raking the zen garden and creating patterns in the sand.
-Extreme close up of a 24 year old woman's eye blinking, standing in Marrakech during magic hour, cinematic film shot in 70mm, depth of field,vivid colors, cinematic.
-A cartoon kangaroo disco dances.
-A beautiful homemade video showing the people of Lagos, Nigeria in the year 2056. Shot with a mobile phone camera.
-A cat waking up its sleeping owner demanding breakfast.The owner tries to ignore the cat, but the cat tries new tactics and finally the owner pulls out a secret stash of treats from under the pillow to hold the cat off a little longer.
-Borneo wildlife on the Kinabatangan River
-A Chinese Lunar New Year celebration video with Chinese Dragon.
-The camera follows behind a white vintage SUv with a black roof rack as it speeds up a steep dirt road surrounded by pine trees on a steep mountain slope, dust kicks up from it's tires, the sunlight shines on the Suv as it speeds along the dirt road,casting a warm glow over the scene. The dirt road curves gently into the distance, with no other cars orvehicles in sight. The trees on either side of the road are redwoods, with patches of greenery scattered throughout. The car is seen from the rear following the curve with ease, making it seem as if it is on a rugged drive through the rugged terrain. The dirt road itself is surrounded by steep hills and mountains with a clear blue sky above with wispy clouds.
-Reflections in the window of a train traveling through the Tokyo suburbs.
-A drone camera circles around a beautiful historic church built on a rocky outcropping along the Amalfi Coast, the view showcases historic and magnificent architectural details and tiered pathways and patios, waves are seen crashing against the rocks below as the view overlooks the horizon of the coastal waters and hilly landscapes of the Amalfi Coast ltaly, several distant people are seen walking and enjoying vistas on patios of the dramatic ocean views, the warm glow of the afternoon sun creates a magical and romantic feeling to the scene, the view is stunning captured with beautiful photography
-A large orange octopus is seen resting on the bottom of the ocean floor, blending in with the sandy and rocky terrain. lts tentacles are spread out around its body, and its eyes are closed. The octopus is unaware of a king crab that is crawling towards it from behind a rock,its claws raised and ready to attack. The crab is brown and spiny,with long legs and antennae. The scene is captured from a wide angle,showing the vastness and depth of the ocean. The wateris clear and blue, with rays of sunlight filtering through. The shot is sharp and crisp, with a high dynamic range. The octopus and the crab are in focus, while the background is slightly blurred,creating a depth of field effect.
-A flock of paper airplanes flutters through a dense jungle,weaving around trees as if they were migrating birds.
-A beautiful silhouette animation shows a wolf howling at the moon,feeling lonely, untilit finds its pack.
-New York City submerged like Atlantis.Fish,whales,sea turtles and sharks swim through the streets of New York.
-A litter of golden retriever puppies playing in the snow.Their heads pop out of the snow, covered in.
-Tour of an art gallery with many beautiful works of art in different styles.
-Beautiful, snowy Tokyo city is bustling. The camera moves through the bustling city street, following several people enjoying the beautiful snowy weather and shopping at nearby stalls. Gorgeous sakura petals are flying through the wind along with snowflakes.
-A stop motion animation of a flower growing out of the windowsill of a suburban house.
-The story of a robot's life in a cyberpunk setting.
-An extreme close-up of an gray-haired man with a beard in his 60s, he is deep in thought pondering the history of the universe as he sits at a cafe in Paris, his eyes focus on people offscreen as they walk as he sits mostly motionless, he is dressed in a wool coat suit coat with a button-down shirt, he wears a brown beret and glasses and has a very professorial appearance, and the end he offers a subtle closed-mouth smile as if he found the answer to the mystery of life, the lighting is very cinematic with the golden light and the Parisian streets and city in the background, depth of field, cinematic 35mm film.
-Basketball through hoop then explodes
-Archeologists discovera generic plastic chairin the desert,excavating and dusting it with great care
-A grandmother with neatly combed grey hair stands behind a colorful birthday cake with numerous candles at a wood dining room table,expression is one of pure joy and happines with a happy glow in her eye. She leans forward and blows out the candles with a gentle puff, the cake has pink frosting and sprinkles and the candles cease to flicker,the grandmotherwears a light blue blouse adorned with floral patterns,several happy friends and family sitting at the table can be seen celebrating,out of focus.The scene is beautifully captured, cinematic, showing a 3/4 view of the grandmother and the dining room. Warm color tones and soft lighting enhance the mood
-Step-printing scene of a person running, cinematic film shot in 35mm
-Five gray wolf pups frolicking and chasing each other around a remote gravel road, surrounded by grass. The pups run and leap, chasing each other, and nipping at each other, playing.
-Tiltshift of a construction site filled with workers, equipment, and heavy machinery.
-A giant, towering cloud in the shape of a man looms overthe earth. The cloud man shoots lighting bolts down to the earth.
-A Samoyed and a Golden Retriever dog are playfully romping through a futuristic neon city at night. The neon lights emitted from the nearby buildings glistens off of their fur.
-The Glenfinnan Viaduct is a historic railway bridge in Scotland, UK, that crosses over the west highland line between the towns of Mallaig and Fort Wiliam. It is a stunning sight as a steam train leaves the bridge, traveling over the arch-covered viaduct. The landscape is dotted with lush greenery and rocky mountains, creating a picturesque backdrop forthe train journey. The sky is blue and the sun is shining,making for a beautiful day to explore this majestic spot.
-The camera directly faces colorful buildings in Burano ltaly. An adorable dalmation looks through a window on a building on the ground floor. Many people are walking and cycling along the canal streets in front of the buildings.
-An adorable happy otter confidently stands on a surfboard wearing a yellow lifejacket, riding along turquoise tropical waters near lush tropical islands,3D digital render art style.
-This close-up shot of a chameleon showcases its striking color changing capabilities.The background is blurred, drawing attention to the animals striking appearance.
-A corgi vlogging itself in tropical Maui.
-A white and orange tabby cat is seen happily darting through a dense garden, as if chasing something.Its eyes are wide and happy as it jogs forward, scanning the branches, flowers, and leaves as it walks. The path is narrow as it makes its way between all the plants. the scene is captured from a ground-level angle, following the cat closely, giving a low and intimate perspective. The image is cinematic with warm tones and a grainy texture. The scattered daylight between the leaves and plants above creates awarm contrast, accentuating the cat's orange fur. The shot is clear and sharp, with a shallow depth of field.
-Aerial view of Santorini during the blue hour, showcasing the stunning architecture of white Cycladic buildings with blue domes. The caldera views are breathtaking,and the lighting creates a beautiful, serene atmosphere.
-Tiltshift of a construction site filled with workers, equipment, and heavy machinery.

examples/veo.txt

@@ -1,19 +0,0 @@
-A lone cowboy rides his horse across an open plain at beautiful sunset, soft light, warm colors
-A fast-tracking shot down an suburban residential street lined with trees. Daytime with a clear blue sky. Saturated colors, high contrast
-Extreme close-up of chicken and green pepper kebabs grilling on a barbeque with flames. Shallow focus and light smoke. vivid colours
-Timelapse of the northern lights dancing across the Arctic sky, stars twinkling, snow-covered landscape
-An aerial shot of a lighthouse standing tall on a rocky cliff, its beacon cutting through the early dawn, waves crash against the rocks below
-Many spotted jellyfish pulsating under water. Their bodies are transparent and glowing in deep ocean
-Timelapse of a common sunflower opening, dark background
-extreme close-up with a shallow depth of field of a puddle in a street. reflecting a busy futuristic Tokyo city with bright neon signs, night, lens flare
-Drone shot along the Hawaii jungle coastline, sunny day
-Drone shot along the Hawaii jungle coastline, sunny day. Kayaks in the water
-Alpacas wearing knit wool sweaters, graffiti background, sunglasses
-Alpacas dancing to the beat
-A fast-tracking shot through a bustling dystopian sprawl with bright neon signs, flying cars and mist, night, lens flare, volumetric lighting. A fast-tracking shot through a futuristic dystopian sprawl with bright neon signs, starships in the sky, night, volumetric lighting. A neon hologram of a car driving at top speed, speed of light, cinematic, incredible details, volumetric lighting. The cars leave the tunnel, back into the real world city Hong Kong.
-A panning shot of a serene mountain landscape, the camera slowly revealing snow-capped peaks, granite rocks and a crystal-clear lake reflecting the sky
-moody shot of a central European alley film noir cinematic black and white high contrast high detail
-Crochet elephant in intricate patterns walking on the savanna
-A Shiba Inu dog wearing a beret and black turtleneck.
-In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, surfing.
-Several giant wooly mammoths approach treading through a snowy meadow, their long wooly fur lightly.

examples/vidu.txt

@@ -1,28 +0,0 @@
-A person clad in a space suit with a helmet and equipped with a chest light and arm device is seen closely examining and interacting with a variety of plants in a lush, indoor botanical setting.
-A desolate lunar landscape with craters and a large moon in the sky transitions to a warmly lit interior of a spacecraft-like structure where a group of people are engaged in various activities.
-An exquisite silverware piece, aesthetically adorned with intricate patterns and scenes, exhibits the detailed artisanship and metallic sheen.
-Under the veil of nightfall, a rose reveals its subtle, exquisite beauty in the gentle moonlight.
-This portrait depicts an orange cat with blue eyes, slowly rotating, inspired by Vermeer ’s ’Girl with a Pearl Earring’. The cat is adorned with pearl earrings and has brown fur styled like a Dutch cap against a black background, illuminated by studio lighting.
-In a studio, there is a painting depicting a ship sailing through the rough sea.
-A red car is stuck in the snow, with the entire vehicle emitting green light and red signal lights flashing on the back. The camera slowly pans around the car.
-A sculptor is intently working on a clay bust, meticulously refining its facial features with precise hand movements.
-Churning ocean waves at night with a lighthouse on the coast create an intense and somewhat foreboding atmosphere. The scene is set under an overcast sky, with the ocean’s dark waters illuminated by natural light, highlighting the white foam of the waves.
-A person holding a dessert with a fluffy layer of whipped cream elegantly drizzled with smooth chocolate sauce. As a dollop of cream falls, a mini polar bear appears, with floating icebergs nearby, set against a serene blue backdrop.
-A large sailing ship sails slowly through the fog.
-An elderly man with a white beard is seated in a room filled with wooden bookshelves, brimming with old books. He is dressed in a dark suit and tie, and he is engrossed in reading a large book. The room is bathed in the warm glow of sunlight streaming through a window, creating a serene and contemplative atmosphere.
-An animated hedgehog with distinctive spiky hair and large eyes is seen exploring a lush, grassy environment.
-A man wearing a hat and a dark suit walks from the corridor towards the room. The lighting casts a bluish tint over the scene, creating a suspenseful atmosphere.
-A rustic wooden cabin nestles by the shore of a clear, sunlit lake, surrounded by verdant trees and mountains. The water is calm, reflecting the sky above, with a few clouds scattered across it. Sailboats and kayaks are moored on the lake, inviting leisure and tranquility.
-A man and a woman are sharing a close and affectionate interaction in an indoor setting that suggests a romantic ambiance.
-An elderly woman with white hair and a lined face is seated inside an older model car, looking out through the side window with a contemplative or mildly sad expression.
-A couple about to get divorced sat awkwardly in the waiting room.
-Audience members in a theater are captured in a series of medium shots, with a young man and woman in formal attire centrally positioned and illuminated by a spotlight effect.
-A painting of a boat on water comes to life, with waves crashing and the boat becoming submerged.
-An animated rabbit in a playful pink snowboarding outfit is carving its way down a snowy mountain slope under a clear blue sky.
-A model train with a blue engine is seen traveling through a meticulously crafted miniature landscape. The train is pulling several red and cream-colored passenger cars along a track that winds through a rural or suburban setting with small-scale houses, verdant trees, and miniature waterfalls.
-The camera rotates around a large stack of vintage televisions all showing different programs — 1950s sci-fi movies, horror movies, news, static, a 1970s sitcom, etc, set inside a large New York museum gallery.
-During the day, a white car drove towards me and splashed water as it passed by a pond, realistic visual style.
-During the day, a red car drove towards me and splashed water as it passed by a pond, realistic visual style.
-During the day, a white car drove towards me and splashed water as it passed by a pond, anime style.
-A pink chrysanthemum flower with intricate petals is the focal point, resting on a wooden surface in an indoor setting.
-A serene mountainous landscape bathed in the warm glow of sunset or twilight, with snow-capped peaks rising above the green vegetation-covered slopes. A calm body of water rests in the foreground, reflecting the sky above, which is dotted with clouds tinged with pink and orange hues.

opensora/sample/sample_t2v.py

@@ -33,23 +33,20 @@ def main(args):
     torch.set_grad_enabled(False)
     device = "cuda" if torch.cuda.is_available() else "cpu"
 
-    # vae = getae_wrapper(args.ae)(args.model_path, subfolder="vae", cache_dir='cache_dir').to(device, dtype=torch.float16)
-    vae = getae_wrapper(args.ae)(args.ae_path).to(device, dtype=torch.float16)
+    vae = getae_wrapper(args.ae)(args.model_path, subfolder="vae", cache_dir=args.cache_dir).to(device, dtype=torch.float16)
+    # vae = getae_wrapper(args.ae)(args.ae_path).to(device, dtype=torch.float16)
     if args.enable_tiling:
         vae.vae.enable_tiling()
         vae.vae.tile_overlap_factor = args.tile_overlap_factor
     vae.vae_scale_factor = ae_stride_config[args.ae]
     # Load model:
-    # transformer_model = LatteT2V.from_pretrained(args.model_path, subfolder=args.version, cache_dir=args.cache_dir, torch_dtype=torch.float16).to(device)
-    transformer_model = LatteT2V.from_pretrained(args.model_path, low_cpu_mem_usage=False, device_map=None, torch_dtype=torch.float16).to(device)
+    transformer_model = LatteT2V.from_pretrained(args.model_path, subfolder=args.version, cache_dir=args.cache_dir, torch_dtype=torch.float16).to(device)
+    # transformer_model = LatteT2V.from_pretrained(args.model_path, low_cpu_mem_usage=False, device_map=None, torch_dtype=torch.float16).to(device)
     
     transformer_model.force_images = args.force_images
     tokenizer = T5Tokenizer.from_pretrained(args.text_encoder_name, cache_dir=args.cache_dir)
     text_encoder = T5EncoderModel.from_pretrained(args.text_encoder_name, cache_dir=args.cache_dir, torch_dtype=torch.float16).to(device)
 
-    # video_length, image_size = transformer_model.config.video_length, args.image_size
-    # latent_size = (image_size // ae_stride_config[args.ae][1], image_size // ae_stride_config[args.ae][2])
-    # vae.latent_size = latent_size
     if args.force_images:
         ext = 'jpg'
     else:
@@ -146,7 +143,7 @@ def main(args):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--model_path", type=str, default='LanguageBind/Open-Sora-Plan-v1.0.0')
-    parser.add_argument("--version", type=str, default=None, choices=[None, '65x512x512', '65x256x256', '17x256x256'])
+    parser.add_argument("--version", type=str, default=None, choices=[None, '65x512x512', '221x512x512', '513x512x512'])
     parser.add_argument("--num_frames", type=int, default=1)
     parser.add_argument("--height", type=int, default=512)
     parser.add_argument("--width", type=int, default=512)

opensora/sample/sample_t2v_json_and_csv.py

@@ -1,268 +0,0 @@
-import os
-import torch
-import argparse
-import torchvision
-
-from diffusers.schedulers import (DDIMScheduler, DDPMScheduler, PNDMScheduler,
-                                  EulerDiscreteScheduler, DPMSolverMultistepScheduler,
-                                  HeunDiscreteScheduler, EulerAncestralDiscreteScheduler,
-                                  DEISMultistepScheduler, KDPM2AncestralDiscreteScheduler)
-from diffusers.schedulers.scheduling_dpmsolver_singlestep import DPMSolverSinglestepScheduler
-from diffusers.models import AutoencoderKL, AutoencoderKLTemporalDecoder
-from omegaconf import OmegaConf
-from transformers import T5EncoderModel, T5Tokenizer, AutoTokenizer
-
-import os, sys
-
-from opensora.models.ae import ae_stride_config, getae, getae_wrapper
-from opensora.models.ae.videobase import CausalVQVAEModelWrapper, CausalVAEModelWrapper
-from opensora.models.diffusion.latte.modeling_latte import LatteT2V
-from opensora.models.text_encoder import get_text_enc
-from opensora.utils.utils import save_video_grid
-
-sys.path.append(os.path.split(sys.path[0])[0])
-from pipeline_videogen import VideoGenPipeline
-
-import json
-import pandas as pd
-import numpy as np
-import imageio
-
-def main(args):
-    # torch.manual_seed(args.seed)
-    torch.set_grad_enabled(False)
-    device = "cuda" if torch.cuda.is_available() else "cpu"
-    cache_dir = "/remote-home1/ysh/MagicBench/1_comparison/Open-Sora-NUS_batch/ckpts"
-
-    vae = getae(args).to(device, dtype=torch.float16)
-    if args.enable_tiling:
-        vae.vae.enable_tiling()
-        vae.vae.tile_overlap_factor = args.tile_overlap_factor
-    if getae_wrapper(args) == CausalVQVAEModelWrapper or getae_wrapper(args) == CausalVAEModelWrapper:
-        video_length = args.num_frames // ae_stride_config[args.ae][0] + 1
-    else:
-        video_length = args.num_frames // ae_stride_config[args.ae][0]
-
-    # Load model:
-    latent_size = (args.image_size // ae_stride_config[args.ae][1], args.image_size // ae_stride_config[args.ae][2])
-    args.latent_size = latent_size
-    vae.latent_size = latent_size
-    transformer_model = LatteT2V.from_pretrained(args.ckpt, subfolder="model", torch_dtype=torch.float16).to(device)
-
-    tokenizer = T5Tokenizer.from_pretrained(args.text_encoder_name, cache_dir=cache_dir)
-    text_encoder = T5EncoderModel.from_pretrained(args.text_encoder_name, cache_dir=cache_dir, torch_dtype=torch.float16).to(device)
-
-    transformer_model.eval()  # important!
-
-    # set eval mode
-    transformer_model.eval()
-    vae.eval()
-    text_encoder.eval()
-
-    if args.sample_method == 'DDIM':
-        scheduler = DDIMScheduler()
-    elif args.sample_method == 'EulerDiscrete':
-        scheduler = EulerDiscreteScheduler()
-    elif args.sample_method == 'DDPM':
-        scheduler = DDPMScheduler()
-    elif args.sample_method == 'DPMSolverMultistep':
-        scheduler = DPMSolverMultistepScheduler()
-    elif args.sample_method == 'DPMSolverSinglestep':
-        scheduler = DPMSolverSinglestepScheduler()
-    elif args.sample_method == 'PNDM':
-        scheduler = PNDMScheduler()
-    elif args.sample_method == 'HeunDiscrete':
-        scheduler = HeunDiscreteScheduler()
-    elif args.sample_method == 'EulerAncestralDiscrete':
-        scheduler = EulerAncestralDiscreteScheduler()
-    elif args.sample_method == 'DEISMultistep':
-        scheduler = DEISMultistepScheduler()
-    elif args.sample_method == 'KDPM2AncestralDiscrete':
-        scheduler = KDPM2AncestralDiscreteScheduler()
-    print('videogen_pipeline', device)
-    videogen_pipeline = VideoGenPipeline(vae=vae,
-                                         text_encoder=text_encoder,
-                                         tokenizer=tokenizer,
-                                         scheduler=scheduler,
-                                         transformer=transformer_model).to(device=device)
-    if args.temporal_lora_path:
-        from swift import Swift
-        print("load lora from swift for Latte")
-        Swift.from_pretrained(videogen_pipeline.transformer, args.temporal_lora_path)
-    # videogen_pipeline.enable_xformers_memory_efficient_attention()
-
-    if not os.path.exists(args.save_img_path):
-        os.makedirs(args.save_img_path)
-
-    video_grids = []
-    if args.prompts_file:
-        with open(args.prompts_file, 'r') as file:
-            prompts = file.read().splitlines()
-    else:
-        if not isinstance(args.text_prompt, list):
-            args.text_prompt = [args.text_prompt]
-        prompts = args.text_prompt if args.text_prompt else []
-
-    if args.human:
-            sample_idx = 0  # Initialize sample index
-            while True:
-                prompt = input("Enter your prompt (or type 'exit' to quit): ")
-                if prompt.lower() == "exit":
-                    break
-
-                args.seed = torch.randint(0, 2**32 - 1, (1,)).item()  # Generate a random seed if not provided
-                torch.manual_seed(args.seed)  # Use the seed for torch operations
-                print(f"current seed: {args.seed}")
-                print('Processing the ({}) prompt'.format(prompt))
-                videos = videogen_pipeline(prompt,
-                                            video_length=video_length,
-                                            height=args.image_size,
-                                            width=args.image_size,
-                                            num_inference_steps=args.num_sampling_steps,
-                                            guidance_scale=args.guidance_scale,
-                                            enable_temporal_attentions=True,
-                                            num_images_per_prompt=1,
-                                            mask_feature=True,
-                                            ).video
-                try:
-                    imageio.mimwrite(
-                            os.path.join(
-                                args.save_img_path,
-                                f"{idx}_seed{args.seed}_{prompt.replace(' ', '_')[:40]}_{args.sample_method}_gs{args.guidance_scale}_s{args.num_sampling_steps}.mp4"
-                            ), videos[0],
-                            fps=args.fps, quality=9)  # highest quality is 10, lowest is 0
-                except:
-                    print('Error when saving {}'.format(prompt))
-                    video_grids.append(videos)    
-                sample_idx += 1
-    elif args.run_csv or args.run_json:
-        batch_size = args.batch_size
-
-        assert args.run_csv != args.run_json
-
-        if args.run_csv:
-            print("run csv")
-            output_video_path = f"./MoT-evaluation/"
-            file_path = "/remote-home1/ysh/MagicBench/1_comparison/0_csv_and_json/chunk_csv/split_data_8.csv"
-            data = pd.read_csv(file_path)
-            prompts = data['name'].tolist()
-            videoids = data['videoid'].tolist()
-        else:
-            print("run json")
-            output_video_path = f"./MSRVTT/"
-            file_path = "/remote-home1/ysh/MagicBench/1_comparison/0_csv_and_json/chunk_json/splitted_file_1.json"
-            with open(file_path, 'r') as file:
-                data = json.load(file)
-            prompts = []
-            videoids = []
-            senids = []
-            for item in data:
-                prompts.append(item['caption'])
-                videoids.append(item['video_id'])
-                senids.append(item['sen_id'])
-
-        os.makedirs(output_video_path, exist_ok=True)
-        
-        for i in range(0, len(prompts), batch_size):
-            batch_prompts_raw = prompts[i : i + batch_size]
-            batch_prompts = [prompt for prompt in batch_prompts_raw]
-
-            if args.run_csv or args.run_json:
-                batch_videoids = videoids[i : i + batch_size]
-            if args.run_json:
-                batch_senids = senids[i : i + batch_size]
-
-            args.seed = torch.randint(0, 2**32 - 1, (1,)).item()  # Generate a random seed if not provided
-            torch.manual_seed(args.seed)  # Use the seed for torch operations
-            print(f"current seed: {args.seed}")
-            
-            videos = videogen_pipeline(batch_prompts,
-                                        video_length=video_length,
-                                        height=args.image_size,
-                                        width=args.image_size,
-                                        num_inference_steps=args.num_sampling_steps,
-                                        guidance_scale=args.guidance_scale,
-                                        enable_temporal_attentions=True,
-                                        num_images_per_prompt=1,
-                                        mask_feature=True,
-                                    ).video
-            
-            for idx, sample in enumerate(videos):
-                if args.run_csv:
-                    new_filename = f"{batch_videoids[idx]}.mp4"
-                if args.run_json:
-                    new_filename = f"{batch_videoids[idx]}-{batch_senids[idx]}.mp4"
-                
-                imageio.mimwrite(os.path.join(output_video_path, new_filename), sample, fps=args.fps, quality=9)  # highest quality is 10, lowest is 0
-    else:
-        for idx, prompt in enumerate(prompts):
-            args.seed = torch.randint(0, 2**32 - 1, (1,)).item()  # Generate a random seed if not provided
-            torch.manual_seed(args.seed)  # Use the seed for torch operations
-            print(f"current seed: {args.seed}")
-            print('Processing the ({}) prompt'.format(prompt))
-            videos = videogen_pipeline(prompt,
-                                    video_length=video_length,
-                                    height=args.image_size,
-                                    width=args.image_size,
-                                    num_inference_steps=args.num_sampling_steps,
-                                    guidance_scale=args.guidance_scale,
-                                    enable_temporal_attentions=True,
-                                    num_images_per_prompt=1,
-                                    mask_feature=True,
-                                    ).video
-            try:
-                imageio.mimwrite(
-                    os.path.join(
-                        args.save_img_path,
-                        f"{idx}_seed{args.seed}_{prompt.replace(' ', '_')[:40]}_{args.sample_method}_gs{args.guidance_scale}_s{args.num_sampling_steps}.mp4"
-                    ), videos[0],
-                    fps=args.fps, quality=9)  # highest quality is 10, lowest is 0
-            except:
-                print('Error when saving {}'.format(prompt))
-            video_grids.append(videos)
-
-    video_grids = torch.cat(video_grids, dim=0)
-
-    video_grids = save_video_grid(video_grids)
-
-    # torchvision.io.write_video(args.save_img_path + '_%04d' % args.run_time + '-.mp4', video_grids, fps=6)
-    imageio.mimwrite(os.path.join(args.save_img_path, f'{args.sample_method}_gs{args.guidance_scale}_s{args.num_sampling_steps}.mp4'), video_grids, fps=args.fps, quality=9)
-    print('save path {}'.format(args.save_img_path))
-
-    # save_videos_grid(video, f"./{prompt}.gif")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--ckpt", type=str, default="")
-    parser.add_argument("--model", type=str, default='Latte-XL/122')
-    parser.add_argument("--ae", type=str, default='CausalVAEModel_4x8x8')
-    parser.add_argument("--text_encoder_name", type=str, default='DeepFloyd/t5-v1_1-xxl')
-    parser.add_argument("--save_img_path", type=str, default="./sample_videos/t2v")
-    parser.add_argument("--beta_start", type=float, default=0.0001)
-    parser.add_argument("--image_size", type=int, default=512)
-    parser.add_argument("--beta_end", type=float, default=0.02)
-    parser.add_argument("--guidance_scale", type=float, default=7.5)
-    parser.add_argument("--beta_schedule", type=str, default="linear")
-    parser.add_argument("--variance_type", type=str, default="learned_range")
-    parser.add_argument("--sample_method", type=str, default="PNDM")
-    parser.add_argument("--num_sampling_steps", type=int, default=50)
-    parser.add_argument("--fps", type=int, default=24)
-    parser.add_argument("--num_frames", type=int, default=65)
-    parser.add_argument("--run_time", type=int, default=0)
-    parser.add_argument("--text_prompt", nargs='+')
-
-    parser.add_argument("--temporal_lora_path", type=str, default=None)
-
-    parser.add_argument("--prompts_file", type=str, help="Path to the text file containing prompts, one per line.")
-    parser.add_argument("--seed", type=int, default=None, help="Random seed for reproducibility")
-    parser.add_argument("--human", action="store_true", help="Enable human mode for interactive video generation")
-    parser.add_argument("--run_csv", type=str, default=None)
-    parser.add_argument("--run_json", type=str, default=None)
-    parser.add_argument("--batch_size", type=int, default=1)
-
-    parser.add_argument('--tile_overlap_factor', type=float, default=0.25)
-    parser.add_argument('--enable_tiling', action='store_true')
-    args = parser.parse_args()
-
-    main(args)

scripts/accelerate_configs/hostfile

@@ -6,4 +6,5 @@ node120 slots=8
 node121 slots=8
 node104 slots=8
 node106 slots=8
-node99 slots=8
+node99 slots=8
+node122 slots=8

scripts/accelerate_configs/hostfile_2

@@ -1,5 +0,0 @@
-node121 slots=8
-node104 slots=8
-node106 slots=8
-node99 slots=8
-node122 slots=8

scripts/accelerate_configs/multi_node_example.yaml

@@ -8,8 +8,8 @@ machine_rank: 0
 main_process_ip: 10.0.10.100
 main_process_port: 29502
 main_training_function: main
-num_machines: 9
-num_processes: 72
+num_machines: 10
+num_processes: 80
 rdzv_backend: static
 same_network: true
 tpu_env: []

scripts/accelerate_configs/multi_node_example_2.yaml

@@ -1,18 +0,0 @@
-compute_environment: LOCAL_MACHINE
-distributed_type: DEEPSPEED
-deepspeed_config:
- deepspeed_config_file: scripts/accelerate_configs/zero2.json
- deepspeed_hostfile: scripts/accelerate_configs/hostfile_2
-fsdp_config: {}
-machine_rank: 0
-main_process_ip: 10.0.10.104
-main_process_port: 29503
-main_training_function: main
-num_machines: 5
-num_processes: 40
-rdzv_backend: static
-same_network: true
-tpu_env: []
-tpu_use_cluster: false
-tpu_use_sudo: false
-use_cpu: false

scripts/accelerate_configs/multi_node_example_node106.yaml

@@ -1,18 +0,0 @@
-compute_environment: LOCAL_MACHINE
-distributed_type: DEEPSPEED
-deepspeed_config:
- deepspeed_config_file: scripts/accelerate_configs/zero2.json
- deepspeed_hostfile: scripts/accelerate_configs/hostfile_node106
-fsdp_config: {}
-machine_rank: 0
-main_process_ip: 10.0.10.106
-main_process_port: 29501
-main_training_function: main
-num_machines: 2
-num_processes: 16
-rdzv_backend: static
-same_network: true
-tpu_env: []
-tpu_use_cluster: false
-tpu_use_sudo: false
-use_cpu: false

scripts/text_condition/sample_image.sh

@@ -1,12 +1,16 @@
-CUDA_VISIBLE_DEVICES=0 python opensora/sample/sample_t2v.py \
-    --model_path LanguageBind/Open-Sora-Plan-v1.0.0 \
+CUDA_VISIBLE_DEVICES=4 python opensora/sample/sample_t2v.py \
+    --model_path LanguageBind/Open-Sora-Plan-v1.1.0 \
+    --version 65x512x512 \
+    --num_frames 1 \
+    --height 512 \
+    --width 512 \
+    --cache_dir "./cache_dir" \
     --text_encoder_name DeepFloyd/t5-v1_1-xxl \
     --text_prompt examples/prompt_list_0.txt \
     --ae CausalVAEModel_4x8x8 \
-    --version 65x512x512 \
-    --save_img_path "./sample_images/prompt_list_0" \
-    --fps 24 \
-    --guidance_scale 7.5 \
-    --num_sampling_steps 250 \
-    --enable_tiling \
-    --force_images
+    --ae_path "/remote-home1/yeyang/CausalVAEModel_4x8x8" \
+    --save_img_path "./sample_image_512x512" \
+    --force_images \
+    --guidance_scale 5.0 \
+    --num_sampling_steps 50 \
+    --enable_tiling

scripts/text_condition/sample_video.sh

@@ -1,18 +0,0 @@
-
-
-CUDA_VISIBLE_DEVICES=$1 python opensora/sample/sample_t2v.py \
-    --model_path hw_65/model \
-    --version 65x512x512 \
-    --num_frames 65 \
-    --height $6 \
-    --width $7 \
-    --cache_dir "./cache_dir" \
-    --text_encoder_name DeepFloyd/t5-v1_1-xxl \
-    --text_prompt examples/$2 \
-    --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
-    --save_img_path $3 \
-    --fps 24 \
-    --guidance_scale $4 \
-    --num_sampling_steps $5 \
-    --enable_tiling

scripts/text_condition/sample_video_257.sh

@@ -1,14 +0,0 @@
-CUDA_VISIBLE_DEVICES=2 python opensora/sample/sample_t2v.py \
-    --model_path 257f_512_10node_bs1_lr2e-5_8img/checkpoint-5000/model \
-    --version 65x512x512 \
-    --image_size 512 \
-    --cache_dir "./cache_dir" \
-    --text_encoder_name DeepFloyd/t5-v1_1-xxl \
-    --text_prompt examples/prompt_list_0.txt \
-    --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
-    --save_img_path "./sample_videos_257f_5k_5.5_100s" \
-    --fps 24 \
-    --guidance_scale 5.5 \
-    --num_sampling_steps 100 \
-    --enable_tiling

scripts/text_condition/sample_video_513.sh

@@ -1,7 +1,7 @@
 
 CUDA_VISIBLE_DEVICES=4 python opensora/sample/sample_t2v.py \
-    --model_path 513f_512_9node_bs1_lr2e-5_4img/checkpoint-11500/model \
-    --version 65x512x512 \
+    --model_path LanguageBind/Open-Sora-Plan-v1.1.0 \
+    --version 513x512x512 \
     --num_frames 513 \
     --height 512 \
     --width 512 \
@@ -9,9 +9,9 @@ CUDA_VISIBLE_DEVICES=4 python opensora/sample/sample_t2v.py \
     --text_encoder_name DeepFloyd/t5-v1_1-xxl \
     --text_prompt examples/prompt_list_0.txt \
     --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
-    --save_img_path "./testttt" \
+    --ae_path "/remote-home1/yeyang/CausalVAEModel_4x8x8" \
+    --save_img_path "./sample_video_513x512x512" \
     --fps 24 \
-    --guidance_scale 5.0 \
+    --guidance_scale 7.5 \
     --num_sampling_steps 150 \
     --enable_tiling

scripts/text_condition/sample_video_65.sh

@@ -1,5 +1,5 @@
 CUDA_VISIBLE_DEVICES=4 python opensora/sample/sample_t2v.py \
-    --model_path hw_65/model \
+    --model_path LanguageBind/Open-Sora-Plan-v1.1.0 \
     --version 65x512x512 \
     --num_frames 65 \
     --height 512 \
@@ -8,9 +8,9 @@ CUDA_VISIBLE_DEVICES=4 python opensora/sample/sample_t2v.py \
     --text_encoder_name DeepFloyd/t5-v1_1-xxl \
     --text_prompt examples/prompt_list_0.txt \
     --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
-    --save_img_path "./testttt" \
+    --ae_path "/remote-home1/yeyang/CausalVAEModel_4x8x8" \
+    --save_img_path "./sample_video_65x512x512" \
     --fps 24 \
-    --guidance_scale 10.0 \
-    --num_sampling_steps 50 \
+    --guidance_scale 7.5 \
+    --num_sampling_steps 150 \
     --enable_tiling

scripts/text_condition/train_videoae_129x512x512.sh

@@ -1,46 +0,0 @@
-export WANDB_KEY="953e958793b218efb850fa194e85843e2c3bd88b"
-# export WANDB_MODE="offline"
-export ENTITY="linbin"
-export PROJECT="129f_512_10node_bs1_lr2e-5_4img"
-export HF_DATASETS_OFFLINE=1 
-export TRANSFORMERS_OFFLINE=1
-# export NCCL_IB_DISABLE=1
-export NCCL_DEBUG=INFO
-export NCCL_SOCKET_IFNAME=ibp25s0
-export NCCL_P2P_DISABLE=1
-export PDSH_PATH_TYPE=ssh
-accelerate launch \
-    --config_file scripts/accelerate_configs/multi_node_example.yaml \
-    opensora/train/train_t2v.py \
-    --model LatteT2V-XL/122 \
-    --text_encoder_name DeepFloyd/t5-v1_1-xxl \
-    --cache_dir "./cache_dir" \
-    --dataset t2v \
-    --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
-    --video_data "scripts/train_data/video_data_513.txt" \
-    --image_data "scripts/train_data/image_data.txt" \
-    --sample_rate 1 \
-    --num_frames 129 \
-    --max_image_size 512 \
-    --gradient_checkpointing \
-    --attention_mode xformers \
-    --train_batch_size=2 \
-    --dataloader_num_workers 8 \
-    --gradient_accumulation_steps=1 \
-    --max_train_steps=1000000 \
-    --learning_rate=2e-05 \
-    --lr_scheduler="constant" \
-    --lr_warmup_steps=0 \
-    --mixed_precision="bf16" \
-    --report_to="wandb" \
-    --checkpointing_steps=500 \
-    --output_dir="129f_512_10node_bs1_lr2e-5_4img" \
-    --allow_tf32 \
-    --use_deepspeed \
-    --model_max_length 300 \
-    --use_image_num 4 \
-    --enable_tiling \
-    --pretrained 512_10node_bs2_lr2e-5_4img/checkpoint-94000/model/diffusion_pytorch_model.safetensors \
-    --enable_tracker \
-    --resume_from_checkpoint "latest" 

scripts/text_condition/train_videoae_221x512x512.sh

@@ -1,27 +1,19 @@
 export WANDB_KEY="953e958793b218efb850fa194e85843e2c3bd88b"
-# export WANDB_MODE="offline"
 export ENTITY="linbin"
-export PROJECT="257f_512_10node_bs1_lr2e-5_4img"
-export HF_DATASETS_OFFLINE=1 
-export TRANSFORMERS_OFFLINE=1
-# export NCCL_IB_DISABLE=1
-export NCCL_DEBUG=INFO
-export NCCL_SOCKET_IFNAME=ibp25s0
-export NCCL_P2P_DISABLE=1
-export PDSH_PATH_TYPE=ssh
+export PROJECT="65x512x512_10node_bs1_lr2e-5_4img"
 accelerate launch \
-    --config_file scripts/accelerate_configs/multi_node_example_2.yaml \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
     opensora/train/train_t2v.py \
     --model LatteT2V-XL/122 \
     --text_encoder_name DeepFloyd/t5-v1_1-xxl \
     --cache_dir "./cache_dir" \
     --dataset t2v \
     --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
+    --ae_path "/remote-home1/yeyang/CausalVAEModel_4x8x8" \
     --video_data "scripts/train_data/video_data_513.txt" \
     --image_data "scripts/train_data/image_data.txt" \
     --sample_rate 1 \
-    --num_frames 257 \
+    --num_frames 221 \
     --max_image_size 512 \
     --gradient_checkpointing \
     --attention_mode xformers \
@@ -34,13 +26,13 @@ accelerate launch \
     --lr_warmup_steps=0 \
     --mixed_precision="bf16" \
     --report_to="wandb" \
-    --checkpointing_steps=500 \
-    --output_dir="257f_512_10node_bs1_lr2e-5_4img" \
+    --checkpointing_steps=250 \
+    --output_dir="65x512x512_10node_bs1_lr2e-5_4img" \
     --allow_tf32 \
     --use_deepspeed \
     --model_max_length 300 \
     --use_image_num 4 \
     --enable_tiling \
-    --pretrained 512_10node_bs2_lr2e-5_4img/checkpoint-94000/model/diffusion_pytorch_model.safetensors \
+    --pretrained 65x512x512_10node_bs2_lr2e-5_4img/diffusion_pytorch_model.safetensors \
     --enable_tracker \
     --resume_from_checkpoint "latest" 

scripts/text_condition/train_videoae_513x512x512.sh

@@ -1,23 +1,15 @@
 export WANDB_KEY=""
-export WANDB_MODE="offline"
 export ENTITY="linbin"
-export PROJECT="513f_512_9node_bs1_lr2e-5_4img"
-export HF_DATASETS_OFFLINE=1 
-export TRANSFORMERS_OFFLINE=1
-# export NCCL_IB_DISABLE=1
-# export NCCL_DEBUG=INFO
-export NCCL_SOCKET_IFNAME=ibp25s0
-export NCCL_P2P_DISABLE=1
-export PDSH_PATH_TYPE=ssh
+export PROJECT="513x512x512_10node_bs1_lr2e-5_4img"
 accelerate launch \
-    --config_file scripts/accelerate_configs/multi_node_example.yaml \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
     opensora/train/train_t2v.py \
     --model LatteT2V-XL/122 \
     --text_encoder_name DeepFloyd/t5-v1_1-xxl \
     --cache_dir "./cache_dir" \
     --dataset t2v \
     --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
+    --ae_path "/remote-home1/yeyang/CausalVAEModel_4x8x8" \
     --video_data "scripts/train_data/video_data_513.txt" \
     --image_data "scripts/train_data/image_data.txt" \
     --sample_rate 1 \
@@ -35,12 +27,12 @@ accelerate launch \
     --mixed_precision="bf16" \
     --report_to="wandb" \
     --checkpointing_steps=250 \
-    --output_dir="513f_512_9node_bs1_lr2e-5_4img" \
+    --output_dir="513x512x512_10node_bs1_lr2e-5_4img" \
     --allow_tf32 \
     --use_deepspeed \
     --model_max_length 300 \
     --use_image_num 4 \
     --enable_tiling \
-    --pretrained /dxyl_code02/dev/Open-Sora-Plan/hw_65/model/diffusion_pytorch_model.safetensors \
+    --pretrained 221x512x512_10node_bs1_lr2e-5_4img/diffusion_pytorch_model.safetensors \
     --enable_tracker \
     --resume_from_checkpoint "latest"

scripts/text_condition/train_videoae_65x512x512.sh

@@ -1,23 +1,15 @@
 export WANDB_KEY=""
-# export WANDB_MODE="offline"
 export ENTITY="linbin"
-export PROJECT="512_10node_bs2_lr2e_4img_"
-export HF_DATASETS_OFFLINE=1 
-export TRANSFORMERS_OFFLINE=1
-# export NCCL_IB_DISABLE=1
-export NCCL_DEBUG=INFO
-# export NCCL_SOCKET_IFNAME=ibp25s0
-export NCCL_P2P_DISABLE=1
-export PDSH_PATH_TYPE=ssh
+export PROJECT="65x512x512_10node_bs2_lr2e-5_4img"
 accelerate launch \
-    --config_file scripts/accelerate_configs/multi_node_example.yaml \
+    --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \
     opensora/train/train_t2v.py \
     --model LatteT2V-XL/122 \
     --text_encoder_name DeepFloyd/t5-v1_1-xxl \
     --cache_dir "./cache_dir" \
     --dataset t2v \
     --ae CausalVAEModel_4x8x8 \
-    --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \
+    --ae_path "/remote-home1/yeyang/CausalVAEModel_4x8x8" \
     --video_data "scripts/train_data/video_data.txt" \
     --image_data "scripts/train_data/image_data.txt" \
     --sample_rate 1 \
@@ -35,7 +27,7 @@ accelerate launch \
     --mixed_precision="bf16" \
     --report_to="wandb" \
     --checkpointing_steps=500 \
-    --output_dir="512_10node_bs2_lr2e-5_4img" \
+    --output_dir="65x512x512_10node_bs2_lr2e-5_4img" \
     --allow_tf32 \
     --use_deepspeed \
     --model_max_length 300 \
@@ -43,5 +35,4 @@ accelerate launch \
     --enable_tiling \
     --pretrained t2v.pt \
     --enable_tracker \
-    --resume_from_checkpoint "latest" \
-    --multi_scale
+    --resume_from_checkpoint "latest" 

scripts/train_data/video_data_old.txt

@@ -1 +0,0 @@
-/dxyl_data02/datasets/mixkit,/dxyl_data02/anno_jsons/tmp_video_mixkit_65f_54735.json