[feat] Add Cosmos 2.5 I2W/V2W support (staged pipeline + examples) (#1021)

Co-authored-by: Will Lin <wlsaidhi@gmail.com>

examples/inference/basic/basic_cosmos2_5_i2w.py

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+from fastvideo import VideoGenerator
+from fastvideo.configs.sample import SamplingParam
+
+
+def main():
+    # Point this to your local diffusers model dir (or replace with a HF model ID).
+    model_path = "KyleShao/Cosmos-Predict2.5-2B-Diffusers"
+
+    generator = VideoGenerator.from_pretrained(
+        model_path,
+        num_gpus=1,
+        use_fsdp_inference=False,  # set True if GPU is out of memory
+        dit_cpu_offload=False,
+        vae_cpu_offload=False,
+        text_encoder_cpu_offload=True,
+        pin_cpu_memory=True,
+    )
+
+    sampling_param = SamplingParam.from_pretrained(model_path)
+
+    # image2world example from official repo
+    image_path = "images/bus_terminal.jpg"
+
+    prompt = (
+        "A nighttime city bus terminal gradually shifts from stillness to subtle movement. "
+        "At first, multiple double-decker buses are parked under the glow of overhead lights, "
+        "with a central bus labeled '87D' facing forward and stationary. "
+        "As the video progresses, the bus in the middle moves ahead slowly, its headlights brightening the surrounding area "
+        "and casting reflections onto adjacent vehicles. "
+        "The motion creates space in the lineup, signaling activity within the otherwise quiet station. "
+        "It then comes to a smooth stop, resuming its position in line. "
+        "Overhead signage in Chinese characters remains illuminated, enhancing the vibrant, urban night scene."
+    )
+
+    generator.generate_video(
+        prompt,
+        sampling_param=sampling_param,
+        image_path=str(image_path),
+        num_cond_frames=1,
+        output_path="outputs_video/cosmos2_5_i2w.mp4",
+        save_video=True,
+    )
+
+    generator.shutdown()
+
+
+if __name__ == "__main__":
+    main()
+
+

examples/inference/basic/basic_cosmos2_5_t2w.py

@@ -1,4 +1,5 @@
 from fastvideo import VideoGenerator
+from fastvideo.configs.sample import SamplingParam
 
 
 def main():
@@ -15,19 +16,27 @@ def main():
         pin_cpu_memory=True,
     )
 
+    # Load default sampling parameters (negative_prompt, resolution, steps, etc.)
+    sampling_param = SamplingParam.from_pretrained(model_path)
+
     prompt = (
-       "A high-definition video captures the precision of robotic welding in an industrial setting. The first frame showcases a robotic arm, equipped with a welding torch, positioned over a large metal structure. The welding process is in full swing, with bright sparks and intense light illuminating the scene, creating a vivid display of blue and white hues. A significant amount of smoke billows around the welding area, partially obscuring the view but emphasizing the heat and activity. The background reveals parts of the workshop environment, including a ventilation system and various pieces of machinery, indicating a busy and functional industrial workspace. As the video progresses, the robotic arm maintains its steady position, continuing the welding process and moving to its left. The welding torch consistently emits sparks and light, and the smoke continues to rise, diffusing slightly as it moves upward. The metal surface beneath the torch shows ongoing signs of heating and melting. The scene retains its industrial ambiance, with the welding sparks and smoke dominating the visual field, underscoring the ongoing nature of the welding operation."
+        "A high-definition video captures the precision of robotic welding in an industrial setting. "
+        "The first frame showcases a robotic arm, equipped with a welding torch, positioned over a large metal structure. "
+        "The welding process is in full swing, with bright sparks and intense light illuminating the scene, "
+        "creating a vivid display of blue and white hues. "
+        "A significant amount of smoke billows around the welding area, partially obscuring the view but emphasizing the heat and activity. "
+        "The background reveals parts of the workshop environment, including a ventilation system and various pieces of machinery, "
+        "indicating a busy and functional industrial workspace. "
+        "As the video progresses, the robotic arm maintains its steady position, continuing the welding process and moving to its left. "
+        "The welding torch consistently emits sparks and light, and the smoke continues to rise, diffusing slightly as it moves upward. "
+        "The metal surface beneath the torch shows ongoing signs of heating and melting. "
+        "The scene retains its industrial ambiance, with the welding sparks and smoke dominating the visual field, "
+        "underscoring the ongoing nature of the welding operation."
     )
 
-    video = generator.generate_video(
+    generator.generate_video(
         prompt,
-        negative_prompt="",
-        height=704,
-        width=1280,
-        num_frames=77,
-        num_inference_steps=35,
-        guidance_scale=7.0,
-        fps=24,
+        sampling_param=sampling_param,
         output_path="outputs_video/cosmos2_5_t2w.mp4",
         save_video=True,
     )

examples/inference/basic/basic_cosmos2_5_v2w.py

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: Apache-2.0
+from fastvideo import VideoGenerator
+from fastvideo.configs.sample import SamplingParam
+
+
+def main():
+    # Point this to your local diffusers model dir (or replace with a HF model ID).
+    model_path = "KyleShao/Cosmos-Predict2.5-2B-Diffusers"
+
+    generator = VideoGenerator.from_pretrained(
+        model_path,
+        num_gpus=1,
+        use_fsdp_inference=False,  # set True if GPU is out of memory
+        dit_cpu_offload=False,
+        vae_cpu_offload=False,
+        text_encoder_cpu_offload=True,
+        pin_cpu_memory=True,
+    )
+
+    sampling_param = SamplingParam.from_pretrained(model_path)
+
+    # video2world example from official repo
+    video_path = "videos/robot_pouring.mp4"
+
+    prompt = (
+        "A robotic arm, primarily white with black joints and cables, is shown in a clean, modern indoor setting with a white tabletop. "
+        "The arm, equipped with a gripper holding a small, light green pitcher, is positioned above a clear glass containing a reddish-brown liquid and a spoon. "
+        "The robotic arm is in the process of pouring a transparent liquid into the glass. "
+        "To the left of the pitcher, there is an opened jar with a similar reddish-brown substance visible through its transparent body. "
+        "In the background, a vase with white flowers and a brown couch are partially visible, adding to the contemporary ambiance. "
+        "The lighting is bright, casting soft shadows on the table. "
+        "The robotic arm's movements are smooth and controlled, demonstrating precision in its task. "
+        "As the video progresses, the robotic arm completes the pour, leaving the glass half-filled with the reddish-brown liquid. "
+        "The jar remains untouched throughout the sequence, and the spoon inside the glass remains stationary. "
+        "The other robotic arm on the right side also stays stationary throughout the video. "
+        "The final frame captures the robotic arm with the pitcher finishing the pour, with the glass now filled to a higher level, while the pitcher is slightly tilted but still held securely by the gripper."
+    )
+
+    generator.generate_video(
+        prompt,
+        sampling_param=sampling_param,
+        video_path=str(video_path),
+        num_cond_frames=1,
+        output_path="outputs_video/cosmos2_5_v2w.mp4",
+        save_video=True,
+    )
+
+    generator.shutdown()
+
+
+if __name__ == "__main__":
+    main()
+
+