From de3f7d8a9fb06c3c5b3fcf3bfa679872bd82bdf5 Mon Sep 17 00:00:00 2001 From: root <2267330597@qq.com> Date: Sat, 25 May 2024 02:54:40 +0000 Subject: [PATCH] prepare release --- .gitignore | 9 +- examples/bak.txt | 16 -- examples/rec_imvi_vae.py | 54 +--- examples/sora.txt | 51 ++++ examples/veo.txt | 19 ++ examples/vidu.txt | 28 ++ .../models/diffusion/latte/modeling_latte.py | 10 +- opensora/models/diffusion/latte/modules.py | 2 +- opensora/sample/pipeline_videogen.py | 21 +- opensora/sample/sample_t2v.py | 21 +- opensora/sample/sample_t2v_json_and_csv.py | 268 ++++++++++++++++++ opensora/train/train_t2v.py | 19 +- scripts/accelerate_configs/hostfile | 3 +- scripts/accelerate_configs/hostfile_2 | 8 +- scripts/accelerate_configs/hostfile_node106 | 2 - .../multi_node_example.yaml | 4 +- .../multi_node_example_2.yaml | 10 +- scripts/causalvae/reconstruction.sh | 15 +- scripts/text_condition/sample_video.sh | 18 +- scripts/text_condition/sample_video_257.sh | 14 + scripts/text_condition/sample_video_513.sh | 17 ++ scripts/text_condition/sample_video_65.sh | 16 ++ scripts/text_condition/train_imageae.sh | 34 --- ...2_test.sh => train_videoae_129x512x512.sh} | 22 +- ..._1node.sh => train_videoae_257x512x512.sh} | 14 +- .../train_videoae_513x512x512.sh | 16 +- .../train_videoae_65x512x512.sh | 4 +- scripts/train_data/video_data_257.txt | 3 - 28 files changed, 520 insertions(+), 198 deletions(-) delete mode 100644 examples/bak.txt create mode 100644 examples/sora.txt create mode 100644 examples/veo.txt create mode 100644 examples/vidu.txt create mode 100644 opensora/sample/sample_t2v_json_and_csv.py delete mode 100644 scripts/accelerate_configs/hostfile_node106 mode change 100644 => 100755 scripts/text_condition/sample_video.sh create mode 100644 scripts/text_condition/sample_video_257.sh create mode 100755 scripts/text_condition/sample_video_513.sh create mode 100644 scripts/text_condition/sample_video_65.sh delete mode 100644 scripts/text_condition/train_imageae.sh rename scripts/text_condition/{train_videoae_65x512x512_test.sh => train_videoae_129x512x512.sh} (68%) rename scripts/text_condition/{train_videoae_513x512x512_1node.sh => train_videoae_257x512x512.sh} (84%) delete mode 100644 scripts/train_data/video_data_257.txt diff --git a/.gitignore b/.gitignore index 1bdab5db3..1b643bd69 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,11 @@ caption* revised* 129f* all* -read* \ No newline at end of file +read* +YSH* +*pick* +*ysh* +hw* +257f* +513f* +taming* \ No newline at end of file diff --git a/examples/bak.txt b/examples/bak.txt deleted file mode 100644 index 5d737d5d2..000000000 --- a/examples/bak.txt +++ /dev/null @@ -1,16 +0,0 @@ -A quiet beach at dawn, the waves gently lapping at the shore and the sky painted in pastel hues. -A quiet beach at dawn, the waves softly lapping at the shore, pink and orange hues painting the sky, offering a moment of solitude and reflection. -The majestic beauty of a waterfall cascading down a cliff into a serene lake. -Sunset over the sea. -a cat wearing sunglasses and walking as a lifeguard at pool. -Slow pan upward of blazing oak fire in an indoor fireplace. -Yellow and black tropical fish dart through the sea. -a serene winter scene in a forest. The forest is blanketed in a thick layer of snow, which has settled on the branches of the trees, creating a canopy of white. The trees, a mix of evergreens and deciduous, stand tall and silent, their forms partially obscured by the snow. The ground is a uniform white, with no visible tracks or signs of human activity. The sun is low in the sky, casting a warm glow that contrasts with the cool tones of the snow. The light filters through the trees, creating a soft, diffused illumination that highlights the texture of the snow and the contours of the trees. The overall style of the scene is naturalistic, with a focus on the tranquility and beauty of the winter landscape. -a dynamic interaction between the ocean and a large rock. The rock, with its rough texture and jagged edges, is partially submerged in the water, suggesting it is a natural feature of the coastline. The water around the rock is in motion, with white foam and waves crashing against the rock, indicating the force of the ocean's movement. The background is a vast expanse of the ocean, with small ripples and waves, suggesting a moderate sea state. The overall style of the scene is a realistic depiction of a natural landscape, with a focus on the interplay between the rock and the water. -A serene waterfall cascading down moss-covered rocks, its soothing sound creating a harmonious symphony with nature. -A soaring drone footage captures the majestic beauty of a coastal cliff, its red and yellow stratified rock faces rich in color and against the vibrant turquoise of the sea. Seabirds can be seen taking flight around the cliff's precipices. As the drone slowly moves from different angles, the changing sunlight casts shifting shadows that highlight the rugged textures of the cliff and the surrounding calm sea. The water gently laps at the rock base and the greenery that clings to the top of the cliff, and the scene gives a sense of peaceful isolation at the fringes of the ocean. The video captures the essence of pristine natural beauty untouched by human structures. -The video captures the majestic beauty of a waterfall cascading down a cliff into a serene lake. The waterfall, with its powerful flow, is the central focus of the video. The surrounding landscape is lush and green, with trees and foliage adding to the natural beauty of the scene. The camera angle provides a bird's eye view of the waterfall, allowing viewers to appreciate the full height and grandeur of the waterfall. The video is a stunning representation of nature's power and beauty. -A vibrant scene of a snowy mountain landscape. The sky is filled with a multitude of colorful hot air balloons, each floating at different heights, creating a dynamic and lively atmosphere. The balloons are scattered across the sky, some closer to the viewer, others further away, adding depth to the scene. Below, the mountainous terrain is blanketed in a thick layer of snow, with a few patches of bare earth visible here and there. The snow-covered mountains provide a stark contrast to the colorful balloons, enhancing the visual appeal of the scene. -A serene underwater scene featuring a sea turtle swimming through a coral reef. The turtle, with its greenish-brown shell, is the main focus of the video, swimming gracefully towards the right side of the frame. The coral reef, teeming with life, is visible in the background, providing a vibrant and colorful backdrop to the turtle's journey. Several small fish, darting around the turtle, add a sense of movement and dynamism to the scene. -A snowy forest landscape with a dirt road running through it. The road is flanked by trees covered in snow, and the ground is also covered in snow. The sun is shining, creating a bright and serene atmosphere. The road appears to be empty, and there are no people or animals visible in the video. The style of the video is a natural landscape shot, with a focus on the beauty of the snowy forest and the peacefulness of the road. -The dynamic movement of tall, wispy grasses swaying in the wind. The sky above is filled with clouds, creating a dramatic backdrop. The sunlight pierces through the clouds, casting a warm glow on the scene. The grasses are a mix of green and brown, indicating a change in seasons. The overall style of the video is naturalistic, capturing the beauty of the landscape in a realistic manner. The focus is on the grasses and their movement, with the sky serving as a secondary element. The video does not contain any human or animal elements. \ No newline at end of file diff --git a/examples/rec_imvi_vae.py b/examples/rec_imvi_vae.py index 35315157f..0bffc6eef 100644 --- a/examples/rec_imvi_vae.py +++ b/examples/rec_imvi_vae.py @@ -20,49 +20,6 @@ from opensora.models.ae import getae_wrapper from opensora.dataset.transform import CenterCropVideo, resize from opensora.models.ae.videobase import CausalVAEModel -def process_in_chunks( - video_data: torch.Tensor, - model: torch.nn.Module, - chunk_size: int, - overlap: int, - device: str, -): - assert (chunk_size + overlap - 1) % 4 == 0 - num_frames = video_data.size(2) - output_chunks = [] - - start = 0 - while start < num_frames: - end = min(start + chunk_size, num_frames) - if start + chunk_size + overlap < num_frames: - end += overlap - chunk = video_data[:, :, start:end, :, :] - - with torch.no_grad(): - chunk = chunk.to(device) - latents = model.encode(chunk) - recon_chunk = model.decode(latents.half()).cpu().float() # b t c h w - recon_chunk = recon_chunk.permute(0, 2, 1, 3, 4) - - if output_chunks: - overlap_step = min(overlap, recon_chunk.shape[2]) - overlap_tensor = ( - output_chunks[-1][:, :, -overlap_step:] * 1 / 4 - + recon_chunk[:, :, :overlap_step] * 3 / 4 - ) - output_chunks[-1] = torch.cat( - (output_chunks[-1][:, :, :-overlap], overlap_tensor), dim=2 - ) - if end < num_frames: - output_chunks.append(recon_chunk[:, :, overlap:]) - else: - output_chunks.append(recon_chunk[:, :, :, :, :]) - else: - output_chunks.append(recon_chunk) - start += chunk_size - return torch.cat(output_chunks, dim=2).permute(0, 2, 1, 3, 4) - - def array_to_video(image_array: npt.NDArray, fps: float = 30.0, output_file: str = 'output_video.mp4') -> None: height, width, channels = image_array[0].shape fourcc = cv2.VideoWriter_fourcc(*'mp4v') @@ -166,12 +123,11 @@ def main(args: argparse.Namespace): x_vae = preprocess(read_video(args.video_path, args.num_frames, args.sample_rate), args.resolution, args.crop_size) x_vae = x_vae.to(device, dtype=torch.float16) # b c t h w - if args.enable_time_chunk: - video_recon = process_in_chunks(x_vae, vae, 7, 2, device) - else: - latents = vae.encode(x_vae) - latents = latents.to(torch.float16) - video_recon = vae.decode(latents) # b t c h w + # from tqdm import tqdm + # for i in tqdm(range(10000000)): + latents = vae.encode(x_vae) + latents = latents.to(torch.float16) + video_recon = vae.decode(latents) # b t c h w if video_recon.shape[2] == 1: x = video_recon[0, 0, :, :, :] diff --git a/examples/sora.txt b/examples/sora.txt new file mode 100644 index 000000000..582e24510 --- /dev/null +++ b/examples/sora.txt @@ -0,0 +1,51 @@ +A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage. She wears a black leather jacket, along red dress, and black boots, and carries a black purse. She wears sunglasses and red lipstick. She walks confidently and casually. The street is dampand reflective, creating a mirror effect of thecolorful lights. Many pedestrians walk about. +Several giant wooly mammoths approach treading through a snowy meadow, their long wooly fur lightly blows in the wind as they walk, snow covered tree sand dramatic snow capped mountains in the distance,mid afternoon lightwith wispy cloud sand a sun high in the distance creates a warm glow, the low camera view is stunning capturing the large furry mammal with beautiful photography, depth of field +A movie trailer featuring the adventures ofthe 30 year old spacemanwearing a redwool knitted motorcycle helmet, bluesky, saltdesert, cinematic style, shoton 35mm film, vivid colors.  +Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach.The crashing blue waters create white-tipped waves,while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green +shrubbery covers the cliffs edge. The steep drop from the road down to the beach is adramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway. +Animated scene features a close-up of a short fluffy monster kneeling beside a melting red candle.The art style is 3D and realistic,with a focus on lighting and texture.The mood of the painting is one of wonder and curiosity, as the monster gazes at the flame with wide eyes and +open mouth. lts pose and expression convey a sense of innocence and playfulness, as if it is exploring the world around it for the first time.The use of warm colors and dramatic lighting further enhances the cozy atmosphere of the image. +A gorgeously rendered papercraft world of a coral reef,rife with colorful fish and sea creatures. +This close-up shot of a Victoria crowned pigeon showcases its striking blue plumage and red chest. Its crest is made of delicate, lacy feathers, while its eye is a striking red color. The bird's head is tilted slightly to the side,giving the impression of it looking regal and majestic. The background is blurred,drawing attention to the bird's striking appearance. +Photorealistic closeup video of two pirate ships battling each other as they sail inside a cup of coffee. +A young man at his 20s is sitting on a piece of cloud in the sky, reading a book. +A petri dish with a bamboo forest growing within it that has tiny red pandas running around. +The camera rotates around a large stack of vintage televisions all showing different programs-1950s sci-fi movies, horror movies, news, static, a 1970s sitcom, etc, set inside a large New York museum gallery. +3D animation of a small, round, fluffy creature with big, expressive eyes explores a vibrant, enchanted forest. The creature, a whimsical blend of a rabbit and a squirrel, has soft blue fur and a bushy, striped tail. It hops along a sparkling stream,its eyes wide with wonder. The forest is alive with magical elements: flowers that glow and change colors, trees with leaves in shades of purple and silver, and small floating lights that resemble fireflies. The creature stops to interact playfully with a group of tiny, fairy-like beings dancing around a mushroom ring. The creature looks up in awe at a large, glowing tree that seems to be the heart of the forest. +Historical footage of California during the gold rush. +A close up view of a glass sphere that has a zen garden within it. There is a small dwarf in the sphere who is raking the zen garden and creating patterns in the sand. +Extreme close up of a 24 year old woman's eye blinking, standing in Marrakech during magic hour, cinematic film shot in 70mm, depth of field,vivid colors, cinematic. +A cartoon kangaroo disco dances. +A beautiful homemade video showing the people of Lagos, Nigeria in the year 2056. Shot with a mobile phone camera. +A cat waking up its sleeping owner demanding breakfast.The owner tries to ignore the cat, but the cat tries new tactics and finally the owner pulls out a secret stash of treats from under the pillow to hold the cat off a little longer. +Borneo wildlife on the Kinabatangan River +A Chinese Lunar New Year celebration video with Chinese Dragon. +The camera follows behind a white vintage SUv with a black roof rack as it speeds up a steep dirt road surrounded by pine trees on a steep mountain slope, dust kicks up from it's tires, the sunlight shines on the Suv as it speeds along the dirt road,casting a warm glow over the scene. The dirt road curves gently into the distance, with no other cars orvehicles in sight. The trees on either side of the road are redwoods, with patches of greenery scattered throughout. The car is seen from the rear following the curve with ease, making it seem as if it is on a rugged drive through the rugged terrain. The dirt road itself is surrounded by steep hills and mountains with a clear blue sky above with wispy clouds. +Reflections in the window of a train traveling through the Tokyo suburbs. +A drone camera circles around a beautiful historic church built on a rocky outcropping along the Amalfi Coast, the view showcases historic and magnificent architectural details and tiered pathways and patios, waves are seen crashing against the rocks below as the view overlooks the horizon of the coastal waters and hilly landscapes of the Amalfi Coast ltaly, several distant people are seen walking and enjoying vistas on patios of the dramatic ocean views, the warm glow of the afternoon sun creates a magical and romantic feeling to the scene, the view is stunning captured with beautiful photography +A large orange octopus is seen resting on the bottom of the ocean floor, blending in with the sandy and rocky terrain. lts tentacles are spread out around its body, and its eyes are closed. The octopus is unaware of a king crab that is crawling towards it from behind a rock,its claws raised and ready to attack. The crab is brown and spiny,with long legs and antennae. The scene is captured from a wide angle,showing the vastness and depth of the ocean. The wateris clear and blue, with rays of sunlight filtering through. The shot is sharp and crisp, with a high dynamic range. The octopus and the crab are in focus, while the background is slightly blurred,creating a depth of field effect. +A flock of paper airplanes flutters through a dense jungle,weaving around trees as if they were migrating birds. +A beautiful silhouette animation shows a wolf howling at the moon,feeling lonely, untilit finds its pack. +New York City submerged like Atlantis.Fish,whales,sea turtles and sharks swim through the streets of New York. +A litter of golden retriever puppies playing in the snow.Their heads pop out of the snow, covered in. +Tour of an art gallery with many beautiful works of art in different styles. +Beautiful, snowy Tokyo city is bustling. The camera moves through the bustling city street, following several people enjoying the beautiful snowy weather and shopping at nearby stalls. Gorgeous sakura petals are flying through the wind along with snowflakes. +A stop motion animation of a flower growing out of the windowsill of a suburban house. +The story of a robot's life in a cyberpunk setting. +An extreme close-up of an gray-haired man with a beard in his 60s, he is deep in thought pondering the history of the universe as he sits at a cafe in Paris, his eyes focus on people offscreen as they walk as he sits mostly motionless, he is dressed in a wool coat suit coat with a button-down shirt, he wears a brown beret and glasses and has a very professorial appearance, and the end he offers a subtle closed-mouth smile as if he found the answer to the mystery of life, the lighting is very cinematic with the golden light and the Parisian streets and city in the background, depth of field, cinematic 35mm film. +Basketball through hoop then explodes +Archeologists discovera generic plastic chairin the desert,excavating and dusting it with great care +A grandmother with neatly combed grey hair stands behind a colorful birthday cake with numerous candles at a wood dining room table,expression is one of pure joy and happines with a happy glow in her eye. She leans forward and blows out the candles with a gentle puff, the cake has pink frosting and sprinkles and the candles cease to flicker,the grandmotherwears a light blue blouse adorned with floral patterns,several happy friends and family sitting at the table can be seen celebrating,out of focus.The scene is beautifully captured, cinematic, showing a 3/4 view of the grandmother and the dining room. Warm color tones and soft lighting enhance the mood +Step-printing scene of a person running, cinematic film shot in 35mm +Five gray wolf pups frolicking and chasing each other around a remote gravel road, surrounded by grass. The pups run and leap, chasing each other, and nipping at each other, playing. +Tiltshift of a construction site filled with workers, equipment, and heavy machinery. +A giant, towering cloud in the shape of a man looms overthe earth. The cloud man shoots lighting bolts down to the earth. +A Samoyed and a Golden Retriever dog are playfully romping through a futuristic neon city at night. The neon lights emitted from the nearby buildings glistens off of their fur. +The Glenfinnan Viaduct is a historic railway bridge in Scotland, UK, that crosses over the west highland line between the towns of Mallaig and Fort Wiliam. It is a stunning sight as a steam train leaves the bridge, traveling over the arch-covered viaduct. The landscape is dotted with lush greenery and rocky mountains, creating a picturesque backdrop forthe train journey. The sky is blue and the sun is shining,making for a beautiful day to explore this majestic spot. +The camera directly faces colorful buildings in Burano ltaly. An adorable dalmation looks through a window on a building on the ground floor. Many people are walking and cycling along the canal streets in front of the buildings. +An adorable happy otter confidently stands on a surfboard wearing a yellow lifejacket, riding along turquoise tropical waters near lush tropical islands,3D digital render art style. +This close-up shot of a chameleon showcases its striking color changing capabilities.The background is blurred, drawing attention to the animals striking appearance. +A corgi vlogging itself in tropical Maui. +A white and orange tabby cat is seen happily darting through a dense garden, as if chasing something.Its eyes are wide and happy as it jogs forward, scanning the branches, flowers, and leaves as it walks. The path is narrow as it makes its way between all the plants. the scene is captured from a ground-level angle, following the cat closely, giving a low and intimate perspective. The image is cinematic with warm tones and a grainy texture. The scattered daylight between the leaves and plants above creates awarm contrast, accentuating the cat's orange fur. The shot is clear and sharp, with a shallow depth of field. +Aerial view of Santorini during the blue hour, showcasing the stunning architecture of white Cycladic buildings with blue domes. The caldera views are breathtaking,and the lighting creates a beautiful, serene atmosphere. +Tiltshift of a construction site filled with workers, equipment, and heavy machinery. \ No newline at end of file diff --git a/examples/veo.txt b/examples/veo.txt new file mode 100644 index 000000000..3f1bfcbb2 --- /dev/null +++ b/examples/veo.txt @@ -0,0 +1,19 @@ +A lone cowboy rides his horse across an open plain at beautiful sunset, soft light, warm colors +A fast-tracking shot down an suburban residential street lined with trees. Daytime with a clear blue sky. Saturated colors, high contrast +Extreme close-up of chicken and green pepper kebabs grilling on a barbeque with flames. Shallow focus and light smoke. vivid colours +Timelapse of the northern lights dancing across the Arctic sky, stars twinkling, snow-covered landscape +An aerial shot of a lighthouse standing tall on a rocky cliff, its beacon cutting through the early dawn, waves crash against the rocks below +Many spotted jellyfish pulsating under water. Their bodies are transparent and glowing in deep ocean +Timelapse of a common sunflower opening, dark background +extreme close-up with a shallow depth of field of a puddle in a street. reflecting a busy futuristic Tokyo city with bright neon signs, night, lens flare +Drone shot along the Hawaii jungle coastline, sunny day +Drone shot along the Hawaii jungle coastline, sunny day. Kayaks in the water +Alpacas wearing knit wool sweaters, graffiti background, sunglasses +Alpacas dancing to the beat +A fast-tracking shot through a bustling dystopian sprawl with bright neon signs, flying cars and mist, night, lens flare, volumetric lighting. A fast-tracking shot through a futuristic dystopian sprawl with bright neon signs, starships in the sky, night, volumetric lighting. A neon hologram of a car driving at top speed, speed of light, cinematic, incredible details, volumetric lighting. The cars leave the tunnel, back into the real world city Hong Kong. +A panning shot of a serene mountain landscape, the camera slowly revealing snow-capped peaks, granite rocks and a crystal-clear lake reflecting the sky +moody shot of a central European alley film noir cinematic black and white high contrast high detail +Crochet elephant in intricate patterns walking on the savanna +A Shiba Inu dog wearing a beret and black turtleneck. +In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, surfing. +Several giant wooly mammoths approach treading through a snowy meadow, their long wooly fur lightly. \ No newline at end of file diff --git a/examples/vidu.txt b/examples/vidu.txt new file mode 100644 index 000000000..95ffae2cb --- /dev/null +++ b/examples/vidu.txt @@ -0,0 +1,28 @@ +A person clad in a space suit with a helmet and equipped with a chest light and arm device is seen closely examining and interacting with a variety of plants in a lush, indoor botanical setting. +A desolate lunar landscape with craters and a large moon in the sky transitions to a warmly lit interior of a spacecraft-like structure where a group of people are engaged in various activities. +An exquisite silverware piece, aesthetically adorned with intricate patterns and scenes, exhibits the detailed artisanship and metallic sheen. +Under the veil of nightfall, a rose reveals its subtle, exquisite beauty in the gentle moonlight. +This portrait depicts an orange cat with blue eyes, slowly rotating, inspired by Vermeer ’s ’Girl with a Pearl Earring’. The cat is adorned with pearl earrings and has brown fur styled like a Dutch cap against a black background, illuminated by studio lighting. +In a studio, there is a painting depicting a ship sailing through the rough sea. +A red car is stuck in the snow, with the entire vehicle emitting green light and red signal lights flashing on the back. The camera slowly pans around the car. +A sculptor is intently working on a clay bust, meticulously refining its facial features with precise hand movements. +Churning ocean waves at night with a lighthouse on the coast create an intense and somewhat foreboding atmosphere. The scene is set under an overcast sky, with the ocean’s dark waters illuminated by natural light, highlighting the white foam of the waves. +A person holding a dessert with a fluffy layer of whipped cream elegantly drizzled with smooth chocolate sauce. As a dollop of cream falls, a mini polar bear appears, with floating icebergs nearby, set against a serene blue backdrop. +A large sailing ship sails slowly through the fog. +An elderly man with a white beard is seated in a room filled with wooden bookshelves, brimming with old books. He is dressed in a dark suit and tie, and he is engrossed in reading a large book. The room is bathed in the warm glow of sunlight streaming through a window, creating a serene and contemplative atmosphere. +An animated hedgehog with distinctive spiky hair and large eyes is seen exploring a lush, grassy environment. +A man wearing a hat and a dark suit walks from the corridor towards the room. The lighting casts a bluish tint over the scene, creating a suspenseful atmosphere. +A rustic wooden cabin nestles by the shore of a clear, sunlit lake, surrounded by verdant trees and mountains. The water is calm, reflecting the sky above, with a few clouds scattered across it. Sailboats and kayaks are moored on the lake, inviting leisure and tranquility. +A man and a woman are sharing a close and affectionate interaction in an indoor setting that suggests a romantic ambiance. +An elderly woman with white hair and a lined face is seated inside an older model car, looking out through the side window with a contemplative or mildly sad expression. +A couple about to get divorced sat awkwardly in the waiting room. +Audience members in a theater are captured in a series of medium shots, with a young man and woman in formal attire centrally positioned and illuminated by a spotlight effect. +A painting of a boat on water comes to life, with waves crashing and the boat becoming submerged. +An animated rabbit in a playful pink snowboarding outfit is carving its way down a snowy mountain slope under a clear blue sky. +A model train with a blue engine is seen traveling through a meticulously crafted miniature landscape. The train is pulling several red and cream-colored passenger cars along a track that winds through a rural or suburban setting with small-scale houses, verdant trees, and miniature waterfalls. +The camera rotates around a large stack of vintage televisions all showing different programs — 1950s sci-fi movies, horror movies, news, static, a 1970s sitcom, etc, set inside a large New York museum gallery. +During the day, a white car drove towards me and splashed water as it passed by a pond, realistic visual style. +During the day, a red car drove towards me and splashed water as it passed by a pond, realistic visual style. +During the day, a white car drove towards me and splashed water as it passed by a pond, anime style. +A pink chrysanthemum flower with intricate petals is the focal point, resting on a wooden surface in an indoor setting. +A serene mountainous landscape bathed in the warm glow of sunset or twilight, with snow-capped peaks rising above the green vegetation-covered slopes. A calm body of water rests in the foreground, reflecting the sky above, which is dotted with clouds tinged with pink and orange hues. \ No newline at end of file diff --git a/opensora/models/diffusion/latte/modeling_latte.py b/opensora/models/diffusion/latte/modeling_latte.py index 340acb6dc..fa8ed7192 100644 --- a/opensora/models/diffusion/latte/modeling_latte.py +++ b/opensora/models/diffusion/latte/modeling_latte.py @@ -85,6 +85,7 @@ class LatteT2V(ModelMixin, ConfigMixin): model_max_length: int = 300, rope_scaling_type: str = 'linear', compress_kv_factor: int = 1, + interpolation_scale_1d: float = None, ): super().__init__() self.use_linear_projection = use_linear_projection @@ -141,10 +142,11 @@ class LatteT2V(ModelMixin, ConfigMixin): # define temporal positional embedding - if self.config.video_length % 2 == 1: - interpolation_scale_1d = (self.config.video_length - 1) // 16 # => 16 (= 16 Latte) has interpolation scale 1 - else: - interpolation_scale_1d = self.config.video_length // 16 # => 16 (= 16 Latte) has interpolation scale 1 + if interpolation_scale_1d is None: + if self.config.video_length % 2 == 1: + interpolation_scale_1d = (self.config.video_length - 1) // 16 # => 16 (= 16 Latte) has interpolation scale 1 + else: + interpolation_scale_1d = self.config.video_length // 16 # => 16 (= 16 Latte) has interpolation scale 1 # interpolation_scale_1d = self.config.video_length // 5 # interpolation_scale_1d = max(interpolation_scale_1d, 1) temp_pos_embed = get_1d_sincos_pos_embed(inner_dim, video_length, interpolation_scale=interpolation_scale_1d) # 1152 hidden size diff --git a/opensora/models/diffusion/latte/modules.py b/opensora/models/diffusion/latte/modules.py index 224a0922d..40ff32e27 100644 --- a/opensora/models/diffusion/latte/modules.py +++ b/opensora/models/diffusion/latte/modules.py @@ -155,7 +155,7 @@ class PatchEmbed(nn.Module): # Interpolate positional embeddings if needed. # (For PixArt-Alpha: https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L162C151-L162C160) if self.height != height or self.width != width: - raise ValueError + # raise ValueError pos_embed = get_2d_sincos_pos_embed( embed_dim=self.pos_embed.shape[-1], grid_size=(height, width), diff --git a/opensora/sample/pipeline_videogen.py b/opensora/sample/pipeline_videogen.py index c48943a25..ce63b2f57 100644 --- a/opensora/sample/pipeline_videogen.py +++ b/opensora/sample/pipeline_videogen.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import math import html import inspect import re @@ -501,15 +501,15 @@ class VideoGenPipeline(DiffusionPipeline): return caption.strip() # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents - def prepare_latents(self, batch_size, num_channels_latents, video_length, height, width, dtype, device, generator, + def prepare_latents(self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None): shape = ( - batch_size, num_channels_latents, video_length, self.vae.latent_size[0], self.vae.latent_size[1]) - if isinstance(generator, list) and len(generator) != batch_size: - raise ValueError( - f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" - f" size of {batch_size}. Make sure the batch size matches the length of the generators." - ) + batch_size, + num_channels_latents, + (math.ceil((int(num_frames) - 1) / self.vae.vae_scale_factor[0]) + 1) if int(num_frames) % 2 == 1 else math.ceil(int(num_frames) / self.vae.vae_scale_factor[0]), + math.ceil(int(height) / self.vae.vae_scale_factor[1]), + math.ceil(int(width) / self.vae.vae_scale_factor[2]), + ) if latents is None: latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) @@ -530,7 +530,7 @@ class VideoGenPipeline(DiffusionPipeline): timesteps: List[int] = None, guidance_scale: float = 4.5, num_images_per_prompt: Optional[int] = 1, - video_length: Optional[int] = None, + num_frames: Optional[int] = None, height: Optional[int] = None, width: Optional[int] = None, eta: float = 0.0, @@ -661,7 +661,7 @@ class VideoGenPipeline(DiffusionPipeline): latents = self.prepare_latents( batch_size * num_images_per_prompt, latent_channels, - video_length, + num_frames, height, width, prompt_embeds.dtype, @@ -738,6 +738,7 @@ class VideoGenPipeline(DiffusionPipeline): if not output_type == 'latents': video = self.decode_latents(latents) + video = video[:, :num_frames, :height, :width] else: video = latents return VideoPipelineOutput(video=video) diff --git a/opensora/sample/sample_t2v.py b/opensora/sample/sample_t2v.py index c2d309293..5c0f8cba6 100644 --- a/opensora/sample/sample_t2v.py +++ b/opensora/sample/sample_t2v.py @@ -38,20 +38,19 @@ def main(args): if args.enable_tiling: vae.vae.enable_tiling() vae.vae.tile_overlap_factor = args.tile_overlap_factor - + vae.vae_scale_factor = ae_stride_config[args.ae] # Load model: # transformer_model = LatteT2V.from_pretrained(args.model_path, subfolder=args.version, cache_dir=args.cache_dir, torch_dtype=torch.float16).to(device) transformer_model = LatteT2V.from_pretrained(args.model_path, low_cpu_mem_usage=False, device_map=None, torch_dtype=torch.float16).to(device) - print(transformer_model.config) + transformer_model.force_images = args.force_images tokenizer = T5Tokenizer.from_pretrained(args.text_encoder_name, cache_dir=args.cache_dir) text_encoder = T5EncoderModel.from_pretrained(args.text_encoder_name, cache_dir=args.cache_dir, torch_dtype=torch.float16).to(device) - video_length, image_size = transformer_model.config.video_length, args.image_size - latent_size = (image_size // ae_stride_config[args.ae][1], image_size // ae_stride_config[args.ae][2]) - vae.latent_size = latent_size + # video_length, image_size = transformer_model.config.video_length, args.image_size + # latent_size = (image_size // ae_stride_config[args.ae][1], image_size // ae_stride_config[args.ae][2]) + # vae.latent_size = latent_size if args.force_images: - video_length = 1 ext = 'jpg' else: ext = 'mp4' @@ -101,9 +100,9 @@ def main(args): for prompt in args.text_prompt: print('Processing the ({}) prompt'.format(prompt)) videos = videogen_pipeline(prompt, - video_length=video_length, - height=image_size, - width=image_size, + num_frames=args.num_frames, + height=args.height, + width=args.width, num_inference_steps=args.num_sampling_steps, guidance_scale=args.guidance_scale, enable_temporal_attentions=not args.force_images, @@ -148,7 +147,9 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--model_path", type=str, default='LanguageBind/Open-Sora-Plan-v1.0.0') parser.add_argument("--version", type=str, default=None, choices=[None, '65x512x512', '65x256x256', '17x256x256']) - parser.add_argument("--image_size", type=int, default=512) + parser.add_argument("--num_frames", type=int, default=1) + parser.add_argument("--height", type=int, default=512) + parser.add_argument("--width", type=int, default=512) parser.add_argument("--cache_dir", type=str, default='./cache_dir') parser.add_argument("--ae", type=str, default='CausalVAEModel_4x8x8') parser.add_argument("--ae_path", type=str, default='CausalVAEModel_4x8x8') diff --git a/opensora/sample/sample_t2v_json_and_csv.py b/opensora/sample/sample_t2v_json_and_csv.py new file mode 100644 index 000000000..c983bca75 --- /dev/null +++ b/opensora/sample/sample_t2v_json_and_csv.py @@ -0,0 +1,268 @@ +import os +import torch +import argparse +import torchvision + +from diffusers.schedulers import (DDIMScheduler, DDPMScheduler, PNDMScheduler, + EulerDiscreteScheduler, DPMSolverMultistepScheduler, + HeunDiscreteScheduler, EulerAncestralDiscreteScheduler, + DEISMultistepScheduler, KDPM2AncestralDiscreteScheduler) +from diffusers.schedulers.scheduling_dpmsolver_singlestep import DPMSolverSinglestepScheduler +from diffusers.models import AutoencoderKL, AutoencoderKLTemporalDecoder +from omegaconf import OmegaConf +from transformers import T5EncoderModel, T5Tokenizer, AutoTokenizer + +import os, sys + +from opensora.models.ae import ae_stride_config, getae, getae_wrapper +from opensora.models.ae.videobase import CausalVQVAEModelWrapper, CausalVAEModelWrapper +from opensora.models.diffusion.latte.modeling_latte import LatteT2V +from opensora.models.text_encoder import get_text_enc +from opensora.utils.utils import save_video_grid + +sys.path.append(os.path.split(sys.path[0])[0]) +from pipeline_videogen import VideoGenPipeline + +import json +import pandas as pd +import numpy as np +import imageio + +def main(args): + # torch.manual_seed(args.seed) + torch.set_grad_enabled(False) + device = "cuda" if torch.cuda.is_available() else "cpu" + cache_dir = "/remote-home1/ysh/MagicBench/1_comparison/Open-Sora-NUS_batch/ckpts" + + vae = getae(args).to(device, dtype=torch.float16) + if args.enable_tiling: + vae.vae.enable_tiling() + vae.vae.tile_overlap_factor = args.tile_overlap_factor + if getae_wrapper(args) == CausalVQVAEModelWrapper or getae_wrapper(args) == CausalVAEModelWrapper: + video_length = args.num_frames // ae_stride_config[args.ae][0] + 1 + else: + video_length = args.num_frames // ae_stride_config[args.ae][0] + + # Load model: + latent_size = (args.image_size // ae_stride_config[args.ae][1], args.image_size // ae_stride_config[args.ae][2]) + args.latent_size = latent_size + vae.latent_size = latent_size + transformer_model = LatteT2V.from_pretrained(args.ckpt, subfolder="model", torch_dtype=torch.float16).to(device) + + tokenizer = T5Tokenizer.from_pretrained(args.text_encoder_name, cache_dir=cache_dir) + text_encoder = T5EncoderModel.from_pretrained(args.text_encoder_name, cache_dir=cache_dir, torch_dtype=torch.float16).to(device) + + transformer_model.eval() # important! + + # set eval mode + transformer_model.eval() + vae.eval() + text_encoder.eval() + + if args.sample_method == 'DDIM': + scheduler = DDIMScheduler() + elif args.sample_method == 'EulerDiscrete': + scheduler = EulerDiscreteScheduler() + elif args.sample_method == 'DDPM': + scheduler = DDPMScheduler() + elif args.sample_method == 'DPMSolverMultistep': + scheduler = DPMSolverMultistepScheduler() + elif args.sample_method == 'DPMSolverSinglestep': + scheduler = DPMSolverSinglestepScheduler() + elif args.sample_method == 'PNDM': + scheduler = PNDMScheduler() + elif args.sample_method == 'HeunDiscrete': + scheduler = HeunDiscreteScheduler() + elif args.sample_method == 'EulerAncestralDiscrete': + scheduler = EulerAncestralDiscreteScheduler() + elif args.sample_method == 'DEISMultistep': + scheduler = DEISMultistepScheduler() + elif args.sample_method == 'KDPM2AncestralDiscrete': + scheduler = KDPM2AncestralDiscreteScheduler() + print('videogen_pipeline', device) + videogen_pipeline = VideoGenPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + transformer=transformer_model).to(device=device) + if args.temporal_lora_path: + from swift import Swift + print("load lora from swift for Latte") + Swift.from_pretrained(videogen_pipeline.transformer, args.temporal_lora_path) + # videogen_pipeline.enable_xformers_memory_efficient_attention() + + if not os.path.exists(args.save_img_path): + os.makedirs(args.save_img_path) + + video_grids = [] + if args.prompts_file: + with open(args.prompts_file, 'r') as file: + prompts = file.read().splitlines() + else: + if not isinstance(args.text_prompt, list): + args.text_prompt = [args.text_prompt] + prompts = args.text_prompt if args.text_prompt else [] + + if args.human: + sample_idx = 0 # Initialize sample index + while True: + prompt = input("Enter your prompt (or type 'exit' to quit): ") + if prompt.lower() == "exit": + break + + args.seed = torch.randint(0, 2**32 - 1, (1,)).item() # Generate a random seed if not provided + torch.manual_seed(args.seed) # Use the seed for torch operations + print(f"current seed: {args.seed}") + print('Processing the ({}) prompt'.format(prompt)) + videos = videogen_pipeline(prompt, + video_length=video_length, + height=args.image_size, + width=args.image_size, + num_inference_steps=args.num_sampling_steps, + guidance_scale=args.guidance_scale, + enable_temporal_attentions=True, + num_images_per_prompt=1, + mask_feature=True, + ).video + try: + imageio.mimwrite( + os.path.join( + args.save_img_path, + f"{idx}_seed{args.seed}_{prompt.replace(' ', '_')[:40]}_{args.sample_method}_gs{args.guidance_scale}_s{args.num_sampling_steps}.mp4" + ), videos[0], + fps=args.fps, quality=9) # highest quality is 10, lowest is 0 + except: + print('Error when saving {}'.format(prompt)) + video_grids.append(videos) + sample_idx += 1 + elif args.run_csv or args.run_json: + batch_size = args.batch_size + + assert args.run_csv != args.run_json + + if args.run_csv: + print("run csv") + output_video_path = f"./MoT-evaluation/" + file_path = "/remote-home1/ysh/MagicBench/1_comparison/0_csv_and_json/chunk_csv/split_data_8.csv" + data = pd.read_csv(file_path) + prompts = data['name'].tolist() + videoids = data['videoid'].tolist() + else: + print("run json") + output_video_path = f"./MSRVTT/" + file_path = "/remote-home1/ysh/MagicBench/1_comparison/0_csv_and_json/chunk_json/splitted_file_1.json" + with open(file_path, 'r') as file: + data = json.load(file) + prompts = [] + videoids = [] + senids = [] + for item in data: + prompts.append(item['caption']) + videoids.append(item['video_id']) + senids.append(item['sen_id']) + + os.makedirs(output_video_path, exist_ok=True) + + for i in range(0, len(prompts), batch_size): + batch_prompts_raw = prompts[i : i + batch_size] + batch_prompts = [prompt for prompt in batch_prompts_raw] + + if args.run_csv or args.run_json: + batch_videoids = videoids[i : i + batch_size] + if args.run_json: + batch_senids = senids[i : i + batch_size] + + args.seed = torch.randint(0, 2**32 - 1, (1,)).item() # Generate a random seed if not provided + torch.manual_seed(args.seed) # Use the seed for torch operations + print(f"current seed: {args.seed}") + + videos = videogen_pipeline(batch_prompts, + video_length=video_length, + height=args.image_size, + width=args.image_size, + num_inference_steps=args.num_sampling_steps, + guidance_scale=args.guidance_scale, + enable_temporal_attentions=True, + num_images_per_prompt=1, + mask_feature=True, + ).video + + for idx, sample in enumerate(videos): + if args.run_csv: + new_filename = f"{batch_videoids[idx]}.mp4" + if args.run_json: + new_filename = f"{batch_videoids[idx]}-{batch_senids[idx]}.mp4" + + imageio.mimwrite(os.path.join(output_video_path, new_filename), sample, fps=args.fps, quality=9) # highest quality is 10, lowest is 0 + else: + for idx, prompt in enumerate(prompts): + args.seed = torch.randint(0, 2**32 - 1, (1,)).item() # Generate a random seed if not provided + torch.manual_seed(args.seed) # Use the seed for torch operations + print(f"current seed: {args.seed}") + print('Processing the ({}) prompt'.format(prompt)) + videos = videogen_pipeline(prompt, + video_length=video_length, + height=args.image_size, + width=args.image_size, + num_inference_steps=args.num_sampling_steps, + guidance_scale=args.guidance_scale, + enable_temporal_attentions=True, + num_images_per_prompt=1, + mask_feature=True, + ).video + try: + imageio.mimwrite( + os.path.join( + args.save_img_path, + f"{idx}_seed{args.seed}_{prompt.replace(' ', '_')[:40]}_{args.sample_method}_gs{args.guidance_scale}_s{args.num_sampling_steps}.mp4" + ), videos[0], + fps=args.fps, quality=9) # highest quality is 10, lowest is 0 + except: + print('Error when saving {}'.format(prompt)) + video_grids.append(videos) + + video_grids = torch.cat(video_grids, dim=0) + + video_grids = save_video_grid(video_grids) + + # torchvision.io.write_video(args.save_img_path + '_%04d' % args.run_time + '-.mp4', video_grids, fps=6) + imageio.mimwrite(os.path.join(args.save_img_path, f'{args.sample_method}_gs{args.guidance_scale}_s{args.num_sampling_steps}.mp4'), video_grids, fps=args.fps, quality=9) + print('save path {}'.format(args.save_img_path)) + + # save_videos_grid(video, f"./{prompt}.gif") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--ckpt", type=str, default="") + parser.add_argument("--model", type=str, default='Latte-XL/122') + parser.add_argument("--ae", type=str, default='CausalVAEModel_4x8x8') + parser.add_argument("--text_encoder_name", type=str, default='DeepFloyd/t5-v1_1-xxl') + parser.add_argument("--save_img_path", type=str, default="./sample_videos/t2v") + parser.add_argument("--beta_start", type=float, default=0.0001) + parser.add_argument("--image_size", type=int, default=512) + parser.add_argument("--beta_end", type=float, default=0.02) + parser.add_argument("--guidance_scale", type=float, default=7.5) + parser.add_argument("--beta_schedule", type=str, default="linear") + parser.add_argument("--variance_type", type=str, default="learned_range") + parser.add_argument("--sample_method", type=str, default="PNDM") + parser.add_argument("--num_sampling_steps", type=int, default=50) + parser.add_argument("--fps", type=int, default=24) + parser.add_argument("--num_frames", type=int, default=65) + parser.add_argument("--run_time", type=int, default=0) + parser.add_argument("--text_prompt", nargs='+') + + parser.add_argument("--temporal_lora_path", type=str, default=None) + + parser.add_argument("--prompts_file", type=str, help="Path to the text file containing prompts, one per line.") + parser.add_argument("--seed", type=int, default=None, help="Random seed for reproducibility") + parser.add_argument("--human", action="store_true", help="Enable human mode for interactive video generation") + parser.add_argument("--run_csv", type=str, default=None) + parser.add_argument("--run_json", type=str, default=None) + parser.add_argument("--batch_size", type=int, default=1) + + parser.add_argument('--tile_overlap_factor', type=float, default=0.25) + parser.add_argument('--enable_tiling', action='store_true') + args = parser.parse_args() + + main(args) \ No newline at end of file diff --git a/opensora/train/train_t2v.py b/opensora/train/train_t2v.py index de054745d..22df3a2bb 100644 --- a/opensora/train/train_t2v.py +++ b/opensora/train/train_t2v.py @@ -74,7 +74,7 @@ def log_validation(args, model, vae, text_encoder, tokenizer, accelerator, weigh for prompt in validation_prompt: logger.info('Processing the ({}) prompt'.format(prompt)) video = videogen_pipeline(prompt, - video_length=args.video_length, + num_frames=args.num_frames, height=args.max_image_size, width=args.max_image_size, num_inference_steps=args.num_sampling_steps, @@ -92,13 +92,13 @@ def log_validation(args, model, vae, text_encoder, tokenizer, accelerator, weigh for tracker in accelerator.trackers: if tracker.name == "tensorboard": np_videos = np.stack([np.asarray(vid) for vid in videos]) - tracker.writer.add_video("validation", np_videos, global_step, fps=10) + tracker.writer.add_video("validation", np_videos, global_step, fps=24) if tracker.name == "wandb": import wandb tracker.log( { "validation": [ - wandb.Video(video, caption=f"{i}: {prompt}", fps=10) + wandb.Video(video, caption=f"{i}: {prompt}", fps=24) for i, (video, prompt) in enumerate(zip(videos, validation_prompt)) ] } @@ -175,6 +175,7 @@ def main(args): text_enc = get_text_warpper(args.text_encoder_name)(args, **kwargs).eval() ae_stride_t, ae_stride_h, ae_stride_w = ae_stride_config[args.ae] + ae.vae_scale_factor = ae_stride_config[args.ae] assert ae_stride_h == ae_stride_w, f"Support only ae_stride_h == ae_stride_w now, but found ae_stride_h ({ae_stride_h}), ae_stride_w ({ae_stride_w})" args.ae_stride_t, args.ae_stride_h, args.ae_stride_w = ae_stride_t, ae_stride_h, ae_stride_w args.ae_stride = args.ae_stride_h @@ -326,8 +327,6 @@ def main(args): train_dataloader = torch.utils.data.DataLoader( train_dataset, shuffle=True, - pin_memory=True, - # prefetch_factor=1, collate_fn=Collate(args), batch_size=args.train_batch_size, num_workers=args.dataloader_num_workers, @@ -421,11 +420,11 @@ def main(args): if not args.multi_scale: assert torch.all(attn_mask) - x = x.to(accelerator.device, dtype=weight_dtype, non_blocking=True) # B C T+num_images H W, 16 + 4 - attn_mask = attn_mask.to(accelerator.device, non_blocking=True) # B L or B 1+num_images L + x = x.to(accelerator.device, dtype=weight_dtype) # B C T+num_images H W, 16 + 4 + attn_mask = attn_mask.to(accelerator.device) # B L or B 1+num_images L # assert torch.all(attn_mask != 0), 'attn_mask must all 1' - input_ids = input_ids.to(accelerator.device, non_blocking=True) # B L or B 1+num_images L - cond_mask = cond_mask.to(accelerator.device, non_blocking=True) # B L or B 1+num_images L + input_ids = input_ids.to(accelerator.device) # B L or B 1+num_images L + cond_mask = cond_mask.to(accelerator.device) # B L or B 1+num_images L # print('x.shape, attn_mask.shape, input_ids.shape, cond_mask.shape', x.shape, attn_mask.shape, input_ids.shape, cond_mask.shape) with torch.no_grad(): @@ -588,7 +587,7 @@ if __name__ == "__main__": parser.add_argument("--cache_dir", type=str, default='./cache_dir') parser.add_argument("--num_sampling_steps", type=int, default=50) - parser.add_argument('--guidance_scale', type=float, default=10.0) + parser.add_argument('--guidance_scale', type=float, default=5.5) parser.add_argument("--multi_scale", action="store_true") parser.add_argument("--enable_tracker", action="store_true") parser.add_argument("--use_deepspeed", action="store_true") diff --git a/scripts/accelerate_configs/hostfile b/scripts/accelerate_configs/hostfile index dccec836e..0133b6d01 100644 --- a/scripts/accelerate_configs/hostfile +++ b/scripts/accelerate_configs/hostfile @@ -6,5 +6,4 @@ node120 slots=8 node121 slots=8 node104 slots=8 node106 slots=8 -node99 slots=8 -node122 slots=8 \ No newline at end of file +node99 slots=8 \ No newline at end of file diff --git a/scripts/accelerate_configs/hostfile_2 b/scripts/accelerate_configs/hostfile_2 index 0133b6d01..f1acdf9f4 100644 --- a/scripts/accelerate_configs/hostfile_2 +++ b/scripts/accelerate_configs/hostfile_2 @@ -1,9 +1,5 @@ -node117 slots=8 # your server name and GPU in total -node100 slots=8 -node118 slots=8 -node119 slots=8 -node120 slots=8 node121 slots=8 node104 slots=8 node106 slots=8 -node99 slots=8 \ No newline at end of file +node99 slots=8 +node122 slots=8 \ No newline at end of file diff --git a/scripts/accelerate_configs/hostfile_node106 b/scripts/accelerate_configs/hostfile_node106 deleted file mode 100644 index 5f88ffbf1..000000000 --- a/scripts/accelerate_configs/hostfile_node106 +++ /dev/null @@ -1,2 +0,0 @@ -node104 slots=8 -node106 slots=8 \ No newline at end of file diff --git a/scripts/accelerate_configs/multi_node_example.yaml b/scripts/accelerate_configs/multi_node_example.yaml index da100f544..1969786cd 100644 --- a/scripts/accelerate_configs/multi_node_example.yaml +++ b/scripts/accelerate_configs/multi_node_example.yaml @@ -8,8 +8,8 @@ machine_rank: 0 main_process_ip: 10.0.10.100 main_process_port: 29502 main_training_function: main -num_machines: 10 -num_processes: 80 +num_machines: 9 +num_processes: 72 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/accelerate_configs/multi_node_example_2.yaml b/scripts/accelerate_configs/multi_node_example_2.yaml index da100f544..e1ff3104c 100644 --- a/scripts/accelerate_configs/multi_node_example_2.yaml +++ b/scripts/accelerate_configs/multi_node_example_2.yaml @@ -2,14 +2,14 @@ compute_environment: LOCAL_MACHINE distributed_type: DEEPSPEED deepspeed_config: deepspeed_config_file: scripts/accelerate_configs/zero2.json - deepspeed_hostfile: scripts/accelerate_configs/hostfile + deepspeed_hostfile: scripts/accelerate_configs/hostfile_2 fsdp_config: {} machine_rank: 0 -main_process_ip: 10.0.10.100 -main_process_port: 29502 +main_process_ip: 10.0.10.104 +main_process_port: 29503 main_training_function: main -num_machines: 10 -num_processes: 80 +num_machines: 5 +num_processes: 40 rdzv_backend: static same_network: true tpu_env: [] diff --git a/scripts/causalvae/reconstruction.sh b/scripts/causalvae/reconstruction.sh index 0c71d27ea..884e9cba6 100644 --- a/scripts/causalvae/reconstruction.sh +++ b/scripts/causalvae/reconstruction.sh @@ -1,11 +1,10 @@ CUDA_VISIBLE_DEVICES=0 python examples/rec_imvi_vae.py \ - --ae_path "/path/to/CausalVAEModel_4x8x8/" \ - --video_path input.mp4 \ - --rec_path rec.mp4 \ + --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \ + --video_path /dxyl_data02/datasets/mixkit/Airplane/mixkit-sun-rays-over-forest-treetops-515.mp4 \ + --rec_path rec_488_129f_512.mp4 \ --device cuda \ --sample_rate 1 \ - --num_frames 65 \ - --resolution 512 \ - --crop_size 512 \ - --ae CausalVAEModel_4x8x8 \ - --enable_tiling \ No newline at end of file + --num_frames 513 \ + --resolution 256 \ + --crop_size 256 \ + --ae CausalVAEModel_4x8x8 \ No newline at end of file diff --git a/scripts/text_condition/sample_video.sh b/scripts/text_condition/sample_video.sh old mode 100644 new mode 100755 index a88913ef4..3948a64c9 --- a/scripts/text_condition/sample_video.sh +++ b/scripts/text_condition/sample_video.sh @@ -1,14 +1,18 @@ -CUDA_VISIBLE_DEVICES=1 python opensora/sample/sample_t2v.py \ - --model_path 512_10node_bs2_lr2e-5_4img/checkpoint-94000/model \ + + +CUDA_VISIBLE_DEVICES=$1 python opensora/sample/sample_t2v.py \ + --model_path hw_65/model \ --version 65x512x512 \ - --image_size 512 \ + --num_frames 65 \ + --height $6 \ + --width $7 \ --cache_dir "./cache_dir" \ --text_encoder_name DeepFloyd/t5-v1_1-xxl \ - --text_prompt examples/prompt_list_0.txt \ + --text_prompt examples/$2 \ --ae CausalVAEModel_4x8x8 \ --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \ - --save_img_path "./sample_videos_94k_7.5_150s" \ + --save_img_path $3 \ --fps 24 \ - --guidance_scale 7.5 \ - --num_sampling_steps 150 \ + --guidance_scale $4 \ + --num_sampling_steps $5 \ --enable_tiling diff --git a/scripts/text_condition/sample_video_257.sh b/scripts/text_condition/sample_video_257.sh new file mode 100644 index 000000000..4824af6a6 --- /dev/null +++ b/scripts/text_condition/sample_video_257.sh @@ -0,0 +1,14 @@ +CUDA_VISIBLE_DEVICES=2 python opensora/sample/sample_t2v.py \ + --model_path 257f_512_10node_bs1_lr2e-5_8img/checkpoint-5000/model \ + --version 65x512x512 \ + --image_size 512 \ + --cache_dir "./cache_dir" \ + --text_encoder_name DeepFloyd/t5-v1_1-xxl \ + --text_prompt examples/prompt_list_0.txt \ + --ae CausalVAEModel_4x8x8 \ + --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \ + --save_img_path "./sample_videos_257f_5k_5.5_100s" \ + --fps 24 \ + --guidance_scale 5.5 \ + --num_sampling_steps 100 \ + --enable_tiling diff --git a/scripts/text_condition/sample_video_513.sh b/scripts/text_condition/sample_video_513.sh new file mode 100755 index 000000000..c58d920c1 --- /dev/null +++ b/scripts/text_condition/sample_video_513.sh @@ -0,0 +1,17 @@ + +CUDA_VISIBLE_DEVICES=4 python opensora/sample/sample_t2v.py \ + --model_path 513f_512_9node_bs1_lr2e-5_4img/checkpoint-11500/model \ + --version 65x512x512 \ + --num_frames 513 \ + --height 512 \ + --width 512 \ + --cache_dir "./cache_dir" \ + --text_encoder_name DeepFloyd/t5-v1_1-xxl \ + --text_prompt examples/prompt_list_0.txt \ + --ae CausalVAEModel_4x8x8 \ + --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \ + --save_img_path "./testttt" \ + --fps 24 \ + --guidance_scale 5.0 \ + --num_sampling_steps 150 \ + --enable_tiling diff --git a/scripts/text_condition/sample_video_65.sh b/scripts/text_condition/sample_video_65.sh new file mode 100644 index 000000000..d58e5930f --- /dev/null +++ b/scripts/text_condition/sample_video_65.sh @@ -0,0 +1,16 @@ +CUDA_VISIBLE_DEVICES=4 python opensora/sample/sample_t2v.py \ + --model_path hw_65/model \ + --version 65x512x512 \ + --num_frames 65 \ + --height 512 \ + --width 512 \ + --cache_dir "./cache_dir" \ + --text_encoder_name DeepFloyd/t5-v1_1-xxl \ + --text_prompt examples/prompt_list_0.txt \ + --ae CausalVAEModel_4x8x8 \ + --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \ + --save_img_path "./testttt" \ + --fps 24 \ + --guidance_scale 10.0 \ + --num_sampling_steps 50 \ + --enable_tiling diff --git a/scripts/text_condition/train_imageae.sh b/scripts/text_condition/train_imageae.sh deleted file mode 100644 index fff9c8afe..000000000 --- a/scripts/text_condition/train_imageae.sh +++ /dev/null @@ -1,34 +0,0 @@ -export WANDB_KEY="" -export ENTITY="" -export PROJECT="t2v-f16s3-img4-128-imgvae188-bf16-gc-xformers" -accelerate launch \ - --config_file scripts/accelerate_configs/deepspeed_zero2_config.yaml \ - opensora/train/train_t2v.py \ - --model LatteT2V-XL/122 \ - --text_encoder_name DeepFloyd/t5-v1_1-xxl \ - --dataset t2v \ - --ae stabilityai/sd-vae-ft-mse \ - --data_path /remote-home1/dataset/sharegpt4v_path_cap_.json \ - --video_folder /remote-home1/dataset/data_split \ - --sample_rate 1 \ - --num_frames 17 \ - --max_image_size 256 \ - --gradient_checkpointing \ - --attention_mode xformers \ - --train_batch_size=4 \ - --dataloader_num_workers 10 \ - --gradient_accumulation_steps=1 \ - --max_train_steps=1000000 \ - --learning_rate=2e-05 \ - --lr_scheduler="constant" \ - --lr_warmup_steps=0 \ - --mixed_precision="bf16" \ - --report_to="wandb" \ - --checkpointing_steps=500 \ - --output_dir="t2v-f17-256-img4-imagevae488-bf16-ckpt-xformers-bs4-lr2e-5-t5" \ - --allow_tf32 \ - --pretrained t2v.pt \ - --use_deepspeed \ - --model_max_length 300 \ - --use_image_num 4 \ - --use_img_from_vid diff --git a/scripts/text_condition/train_videoae_65x512x512_test.sh b/scripts/text_condition/train_videoae_129x512x512.sh similarity index 68% rename from scripts/text_condition/train_videoae_65x512x512_test.sh rename to scripts/text_condition/train_videoae_129x512x512.sh index 0803dc2fa..2abc89a2c 100644 --- a/scripts/text_condition/train_videoae_65x512x512_test.sh +++ b/scripts/text_condition/train_videoae_129x512x512.sh @@ -1,12 +1,12 @@ -export WANDB_KEY="" +export WANDB_KEY="953e958793b218efb850fa194e85843e2c3bd88b" # export WANDB_MODE="offline" export ENTITY="linbin" -export PROJECT="512_10node_bs2_lr2e_4img_test1node" +export PROJECT="129f_512_10node_bs1_lr2e-5_4img" export HF_DATASETS_OFFLINE=1 export TRANSFORMERS_OFFLINE=1 # export NCCL_IB_DISABLE=1 -# export NCCL_DEBUG=INFO -# export NCCL_SOCKET_IFNAME=ibp25s0 +export NCCL_DEBUG=INFO +export NCCL_SOCKET_IFNAME=ibp25s0 export NCCL_P2P_DISABLE=1 export PDSH_PATH_TYPE=ssh accelerate launch \ @@ -18,29 +18,29 @@ accelerate launch \ --dataset t2v \ --ae CausalVAEModel_4x8x8 \ --ae_path "/dxyl_data02/CausalVAEModel_4x8x8/" \ - --video_data "scripts/train_data/video_data.txt" \ + --video_data "scripts/train_data/video_data_513.txt" \ --image_data "scripts/train_data/image_data.txt" \ --sample_rate 1 \ - --num_frames 65 \ + --num_frames 129 \ --max_image_size 512 \ --gradient_checkpointing \ --attention_mode xformers \ --train_batch_size=2 \ - --dataloader_num_workers 10 \ + --dataloader_num_workers 8 \ --gradient_accumulation_steps=1 \ --max_train_steps=1000000 \ --learning_rate=2e-05 \ --lr_scheduler="constant" \ --lr_warmup_steps=0 \ --mixed_precision="bf16" \ - --report_to="tensorboard" \ + --report_to="wandb" \ --checkpointing_steps=500 \ - --output_dir="512_10node_bs2_lr2e-5_4img_test1node" \ + --output_dir="129f_512_10node_bs1_lr2e-5_4img" \ --allow_tf32 \ --use_deepspeed \ --model_max_length 300 \ --use_image_num 4 \ --enable_tiling \ - --pretrained t2v.pt \ + --pretrained 512_10node_bs2_lr2e-5_4img/checkpoint-94000/model/diffusion_pytorch_model.safetensors \ --enable_tracker \ - --resume_from_checkpoint "latest" + --resume_from_checkpoint "latest" diff --git a/scripts/text_condition/train_videoae_513x512x512_1node.sh b/scripts/text_condition/train_videoae_257x512x512.sh similarity index 84% rename from scripts/text_condition/train_videoae_513x512x512_1node.sh rename to scripts/text_condition/train_videoae_257x512x512.sh index 4e7792013..2e7c94b12 100644 --- a/scripts/text_condition/train_videoae_513x512x512_1node.sh +++ b/scripts/text_condition/train_videoae_257x512x512.sh @@ -1,11 +1,11 @@ export WANDB_KEY="953e958793b218efb850fa194e85843e2c3bd88b" # export WANDB_MODE="offline" export ENTITY="linbin" -export PROJECT="129f_512_10node_bs1_lr2e-5_2img" +export PROJECT="257f_512_10node_bs1_lr2e-5_4img" export HF_DATASETS_OFFLINE=1 export TRANSFORMERS_OFFLINE=1 # export NCCL_IB_DISABLE=1 -# export NCCL_DEBUG=INFO +export NCCL_DEBUG=INFO export NCCL_SOCKET_IFNAME=ibp25s0 export NCCL_P2P_DISABLE=1 export PDSH_PATH_TYPE=ssh @@ -21,7 +21,7 @@ accelerate launch \ --video_data "scripts/train_data/video_data_513.txt" \ --image_data "scripts/train_data/image_data.txt" \ --sample_rate 1 \ - --num_frames 129 \ + --num_frames 257 \ --max_image_size 512 \ --gradient_checkpointing \ --attention_mode xformers \ @@ -34,13 +34,13 @@ accelerate launch \ --lr_warmup_steps=0 \ --mixed_precision="bf16" \ --report_to="wandb" \ - --checkpointing_steps=100 \ - --output_dir="129f_512_10node_bs1_lr2e-5_2img" \ + --checkpointing_steps=500 \ + --output_dir="257f_512_10node_bs1_lr2e-5_4img" \ --allow_tf32 \ --use_deepspeed \ --model_max_length 300 \ - --use_image_num 2 \ + --use_image_num 4 \ --enable_tiling \ --pretrained 512_10node_bs2_lr2e-5_4img/checkpoint-94000/model/diffusion_pytorch_model.safetensors \ --enable_tracker \ - --resume_from_checkpoint "latest" + --resume_from_checkpoint "latest" diff --git a/scripts/text_condition/train_videoae_513x512x512.sh b/scripts/text_condition/train_videoae_513x512x512.sh index 5d984a8df..0d96bba0d 100644 --- a/scripts/text_condition/train_videoae_513x512x512.sh +++ b/scripts/text_condition/train_videoae_513x512x512.sh @@ -1,7 +1,7 @@ export WANDB_KEY="" -# export WANDB_MODE="offline" +export WANDB_MODE="offline" export ENTITY="linbin" -export PROJECT="513f_512_10node_bs1_lr2e-5_2img" +export PROJECT="513f_512_9node_bs1_lr2e-5_4img" export HF_DATASETS_OFFLINE=1 export TRANSFORMERS_OFFLINE=1 # export NCCL_IB_DISABLE=1 @@ -26,21 +26,21 @@ accelerate launch \ --gradient_checkpointing \ --attention_mode xformers \ --train_batch_size=1 \ - --dataloader_num_workers 4 \ + --dataloader_num_workers 8 \ --gradient_accumulation_steps=1 \ --max_train_steps=1000000 \ --learning_rate=2e-05 \ --lr_scheduler="constant" \ --lr_warmup_steps=0 \ --mixed_precision="bf16" \ - --report_to="tensorboard" \ - --checkpointing_steps=500 \ - --output_dir="513f_512_10node_bs1_lr2e-5_2img" \ + --report_to="wandb" \ + --checkpointing_steps=250 \ + --output_dir="513f_512_9node_bs1_lr2e-5_4img" \ --allow_tf32 \ --use_deepspeed \ --model_max_length 300 \ - --use_image_num 2 \ + --use_image_num 4 \ --enable_tiling \ - --pretrained 512_10node_bs2_lr2e-5_4img/checkpoint-101500/model/diffusion_pytorch_model.safetensors \ + --pretrained /dxyl_code02/dev/Open-Sora-Plan/hw_65/model/diffusion_pytorch_model.safetensors \ --enable_tracker \ --resume_from_checkpoint "latest" diff --git a/scripts/text_condition/train_videoae_65x512x512.sh b/scripts/text_condition/train_videoae_65x512x512.sh index c4b223b30..fef1de53d 100644 --- a/scripts/text_condition/train_videoae_65x512x512.sh +++ b/scripts/text_condition/train_videoae_65x512x512.sh @@ -5,7 +5,7 @@ export PROJECT="512_10node_bs2_lr2e_4img_" export HF_DATASETS_OFFLINE=1 export TRANSFORMERS_OFFLINE=1 # export NCCL_IB_DISABLE=1 -# export NCCL_DEBUG=INFO +export NCCL_DEBUG=INFO # export NCCL_SOCKET_IFNAME=ibp25s0 export NCCL_P2P_DISABLE=1 export PDSH_PATH_TYPE=ssh @@ -34,7 +34,7 @@ accelerate launch \ --lr_warmup_steps=0 \ --mixed_precision="bf16" \ --report_to="wandb" \ - --checkpointing_steps=200 \ + --checkpointing_steps=500 \ --output_dir="512_10node_bs2_lr2e-5_4img" \ --allow_tf32 \ --use_deepspeed \ diff --git a/scripts/train_data/video_data_257.txt b/scripts/train_data/video_data_257.txt deleted file mode 100644 index b42b6ea8d..000000000 --- a/scripts/train_data/video_data_257.txt +++ /dev/null @@ -1,3 +0,0 @@ -/dxyl_data02/datasets/pixabay_v2,/dxyl_data02/anno_jsons/video_pixabay_513f_51483.json -/dxyl_data02/datasets/mixkit,/dxyl_data02/anno_jsons/video_mixkit_513f_1997.json -/dxyl_data02/datasets/pexels,/dxyl_data02/anno_jsons/video_pexel_513f_271782.json