update gamecraft
This commit is contained in:
Mihir Jagtap
@@ -0,0 +1,346 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""
|
||||
Basic example for HunyuanGameCraft video generation using FastVideo.
|
||||
|
||||
HunyuanGameCraft is a camera-controllable video generation model that uses
|
||||
Plücker coordinate representation (6D ray format) for camera pose conditioning.
|
||||
|
||||
This example demonstrates:
|
||||
1. Text-to-video generation with camera control
|
||||
2. Using CameraNet for processing camera states
|
||||
3. Integration with the FastVideo pipeline
|
||||
|
||||
Usage:
|
||||
python basic_hunyuangamecraft.py --prompt "A game scene" --output video_output.mp4
|
||||
|
||||
Note: This requires converted HunyuanGameCraft weights. If using official weights,
|
||||
run the conversion script first:
|
||||
python scripts/checkpoint_conversion/convert_gamecraft_to_fastvideo.py \
|
||||
--source /path/to/official_weights/gamecraft_model_states.pt \
|
||||
--output converted_weights/HunyuanGameCraft
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from fastvideo import VideoGenerator
|
||||
from fastvideo.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def create_camera_trajectory(
|
||||
num_frames: int,
|
||||
height: int,
|
||||
width: int,
|
||||
trajectory_type: str = "forward",
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Create camera trajectory using Plücker coordinates.
|
||||
|
||||
Plücker coordinates represent a 3D line (ray) using 6 numbers:
|
||||
- First 3: Direction vector (d)
|
||||
- Last 3: Moment vector (m = p × d, where p is a point on the line)
|
||||
|
||||
Args:
|
||||
num_frames: Number of frames in the video
|
||||
height: Height of the video in pixels
|
||||
width: Width of the video in pixels
|
||||
trajectory_type: Type of camera trajectory
|
||||
- "forward": Camera moves forward
|
||||
- "backward": Camera moves backward
|
||||
- "left": Camera moves left
|
||||
- "right": Camera moves right
|
||||
- "stationary": Camera stays still
|
||||
- "orbit": Camera orbits around center
|
||||
|
||||
Returns:
|
||||
Camera states tensor of shape [num_frames, 6, height, width]
|
||||
"""
|
||||
# Create a grid of pixel coordinates
|
||||
y_coords = torch.linspace(-1, 1, height)
|
||||
x_coords = torch.linspace(-1, 1, width)
|
||||
grid_y, grid_x = torch.meshgrid(y_coords, x_coords, indexing='ij')
|
||||
|
||||
camera_states = torch.zeros(num_frames, 6, height, width)
|
||||
|
||||
for frame_idx in range(num_frames):
|
||||
t = frame_idx / max(num_frames - 1, 1) # Normalized time [0, 1]
|
||||
|
||||
# Define camera movement based on trajectory type
|
||||
if trajectory_type == "forward":
|
||||
# Camera moves forward (positive z direction)
|
||||
direction = torch.tensor([0.0, 0.0, 1.0])
|
||||
position = torch.tensor([0.0, 0.0, -3.0 + t * 2.0])
|
||||
elif trajectory_type == "backward":
|
||||
# Camera moves backward (negative z direction)
|
||||
direction = torch.tensor([0.0, 0.0, -1.0])
|
||||
position = torch.tensor([0.0, 0.0, -1.0 - t * 2.0])
|
||||
elif trajectory_type == "left":
|
||||
# Camera moves left (negative x direction)
|
||||
direction = torch.tensor([-1.0, 0.0, 0.0])
|
||||
position = torch.tensor([1.0 - t * 2.0, 0.0, -2.0])
|
||||
elif trajectory_type == "right":
|
||||
# Camera moves right (positive x direction)
|
||||
direction = torch.tensor([1.0, 0.0, 0.0])
|
||||
position = torch.tensor([-1.0 + t * 2.0, 0.0, -2.0])
|
||||
elif trajectory_type == "orbit":
|
||||
# Camera orbits around center
|
||||
angle = t * 2 * np.pi # Full rotation
|
||||
radius = 3.0
|
||||
position = torch.tensor([
|
||||
radius * np.sin(angle),
|
||||
0.0,
|
||||
-radius * np.cos(angle)
|
||||
])
|
||||
# Direction points toward center
|
||||
direction = -position / position.norm()
|
||||
else: # stationary
|
||||
direction = torch.tensor([0.0, 0.0, 1.0])
|
||||
position = torch.tensor([0.0, 0.0, -3.0])
|
||||
|
||||
# Normalize direction
|
||||
direction = direction / direction.norm()
|
||||
|
||||
# Compute Plücker coordinates for each pixel
|
||||
# For each pixel, we create a ray from the camera through that pixel
|
||||
for y in range(height):
|
||||
for x in range(width):
|
||||
# Pixel direction in camera space (normalized device coordinates)
|
||||
pixel_dir = torch.tensor([
|
||||
grid_x[y, x].item(),
|
||||
grid_y[y, x].item(),
|
||||
1.0
|
||||
])
|
||||
pixel_dir = pixel_dir / pixel_dir.norm()
|
||||
|
||||
# Transform to world space (simplified - assuming identity rotation)
|
||||
world_dir = pixel_dir
|
||||
|
||||
# Moment vector: m = p × d
|
||||
moment = torch.cross(position, world_dir)
|
||||
|
||||
# Store Plücker coordinates: [d, m]
|
||||
camera_states[frame_idx, 0:3, y, x] = world_dir
|
||||
camera_states[frame_idx, 3:6, y, x] = moment
|
||||
|
||||
return camera_states
|
||||
|
||||
|
||||
def create_simple_camera_states(
|
||||
num_frames: int,
|
||||
height: int,
|
||||
width: int,
|
||||
trajectory_type: str = "forward",
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Create simplified camera states for testing.
|
||||
|
||||
This creates camera states using a simpler approach that's more efficient
|
||||
for testing purposes. For production use, you may want to use more
|
||||
sophisticated camera trajectory generation.
|
||||
|
||||
Args:
|
||||
num_frames: Number of frames
|
||||
height: Video height
|
||||
width: Video width
|
||||
trajectory_type: Type of camera movement
|
||||
|
||||
Returns:
|
||||
Camera states tensor of shape [num_frames, 6, height, width]
|
||||
"""
|
||||
camera_states = torch.zeros(num_frames, 6, height, width)
|
||||
|
||||
# Movement vectors for different trajectories
|
||||
movement_configs = {
|
||||
"forward": (0.0, 0.0, 0.1), # +z direction
|
||||
"backward": (0.0, 0.0, -0.1), # -z direction
|
||||
"left": (-0.1, 0.0, 0.0), # -x direction
|
||||
"right": (0.1, 0.0, 0.0), # +x direction
|
||||
"up": (0.0, 0.1, 0.0), # +y direction
|
||||
"down": (0.0, -0.1, 0.0), # -y direction
|
||||
"stationary": (0.0, 0.0, 0.0), # no movement
|
||||
}
|
||||
|
||||
move = movement_configs.get(trajectory_type, (0.0, 0.0, 0.0))
|
||||
|
||||
for frame_idx in range(num_frames):
|
||||
t = frame_idx / max(num_frames - 1, 1)
|
||||
|
||||
# Direction (first 3 channels) - forward direction with slight variation
|
||||
camera_states[frame_idx, 0, :, :] = move[0] * t # x component
|
||||
camera_states[frame_idx, 1, :, :] = move[1] * t # y component
|
||||
camera_states[frame_idx, 2, :, :] = 1.0 + move[2] * t # z component
|
||||
|
||||
# Moment (last 3 channels) - simplified
|
||||
camera_states[frame_idx, 3, :, :] = 0.0
|
||||
camera_states[frame_idx, 4, :, :] = 0.0
|
||||
camera_states[frame_idx, 5, :, :] = 0.0
|
||||
|
||||
return camera_states
|
||||
|
||||
|
||||
# Default settings
|
||||
DEFAULT_MODEL_PATH = "converted_weights/HunyuanGameCraft"
|
||||
DEFAULT_PROMPT = (
|
||||
"A first-person view of walking through a dense forest. "
|
||||
"Sunlight filters through the canopy, creating dappled shadows on the ground. "
|
||||
"The camera moves forward steadily."
|
||||
)
|
||||
DEFAULT_OUTPUT_PATH = "video_samples_hunyuangamecraft"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="HunyuanGameCraft video generation with FastVideo"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-path",
|
||||
type=str,
|
||||
default=DEFAULT_MODEL_PATH,
|
||||
help="Path to converted HunyuanGameCraft weights"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
type=str,
|
||||
default=DEFAULT_PROMPT,
|
||||
help="Text prompt for video generation"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--negative-prompt",
|
||||
type=str,
|
||||
default="",
|
||||
help="Negative prompt"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-path",
|
||||
type=str,
|
||||
default=DEFAULT_OUTPUT_PATH,
|
||||
help="Output video path"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-frames",
|
||||
type=int,
|
||||
default=33,
|
||||
help="Number of frames (default: 33, ~1.3s at 24fps)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--height",
|
||||
type=int,
|
||||
default=704,
|
||||
help="Video height in pixels"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--width",
|
||||
type=int,
|
||||
default=1280,
|
||||
help="Video width in pixels"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-inference-steps",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Number of denoising steps"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--guidance-scale",
|
||||
type=float,
|
||||
default=6.0,
|
||||
help="Classifier-free guidance scale"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=42,
|
||||
help="Random seed for reproducibility"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--trajectory",
|
||||
type=str,
|
||||
default="forward",
|
||||
choices=["forward", "backward", "left", "right", "up", "down", "stationary"],
|
||||
help="Camera trajectory type"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-gpus",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of GPUs to use"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cpu-offload",
|
||||
action="store_true",
|
||||
help="Enable CPU offloading for memory-constrained setups"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print("=" * 60)
|
||||
print("HunyuanGameCraft Video Generation")
|
||||
print("=" * 60)
|
||||
print(f"Model path: {args.model_path}")
|
||||
print(f"Prompt: {args.prompt[:100]}...")
|
||||
print(f"Resolution: {args.height}x{args.width}")
|
||||
print(f"Frames: {args.num_frames}")
|
||||
print(f"Camera trajectory: {args.trajectory}")
|
||||
print(f"Output: {args.output_path}")
|
||||
print("=" * 60)
|
||||
|
||||
# Initialize generator
|
||||
print("\nInitializing VideoGenerator...")
|
||||
generator = VideoGenerator.from_pretrained(
|
||||
args.model_path,
|
||||
num_gpus=args.num_gpus,
|
||||
use_fsdp_inference=args.num_gpus > 1,
|
||||
dit_cpu_offload=args.cpu_offload,
|
||||
vae_cpu_offload=args.cpu_offload,
|
||||
text_encoder_cpu_offload=args.cpu_offload,
|
||||
pin_cpu_memory=True,
|
||||
)
|
||||
|
||||
# Create camera trajectory
|
||||
print(f"\nCreating camera trajectory ({args.trajectory})...")
|
||||
camera_states = create_simple_camera_states(
|
||||
num_frames=args.num_frames,
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
trajectory_type=args.trajectory,
|
||||
)
|
||||
# Add batch dimension
|
||||
camera_states = camera_states.unsqueeze(0) # [1, F, 6, H, W]
|
||||
|
||||
print(f"Camera states shape: {camera_states.shape}")
|
||||
|
||||
# Generate video
|
||||
print("\nGenerating video...")
|
||||
start_time = time.time()
|
||||
|
||||
result = generator.generate_video(
|
||||
prompt=args.prompt,
|
||||
negative_prompt=args.negative_prompt,
|
||||
camera_states=camera_states,
|
||||
num_frames=args.num_frames,
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
guidance_scale=args.guidance_scale,
|
||||
seed=args.seed,
|
||||
output_path=args.output_path,
|
||||
save_video=True,
|
||||
)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
print(f"\nVideo generated successfully!")
|
||||
print(f"Saved to: {args.output_path}")
|
||||
print(f"Generation time: {elapsed:.2f}s")
|
||||
print(f"Frames per second: {args.num_frames / elapsed:.2f}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user