From debaa56f04f48c8a24d85f66acfaad45a809daa0 Mon Sep 17 00:00:00 2001 From: Mihir Jagtap Date: Fri, 6 Feb 2026 08:43:08 -0800 Subject: [PATCH] update gamecraft --- .../inference/basic/basic_hunyuangamecraft.py | 346 +++++++++++++ .../basic/basic_hunyuangamecraft_i2v.py | 473 ++++++++++++++++++ .../configs/models/dits/hunyuangamecraft.py | 15 +- 3 files changed, 830 insertions(+), 4 deletions(-) create mode 100644 examples/inference/basic/basic_hunyuangamecraft.py create mode 100644 examples/inference/basic/basic_hunyuangamecraft_i2v.py diff --git a/examples/inference/basic/basic_hunyuangamecraft.py b/examples/inference/basic/basic_hunyuangamecraft.py new file mode 100644 index 000000000..09bae5f2d --- /dev/null +++ b/examples/inference/basic/basic_hunyuangamecraft.py @@ -0,0 +1,346 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Basic example for HunyuanGameCraft video generation using FastVideo. + +HunyuanGameCraft is a camera-controllable video generation model that uses +Plücker coordinate representation (6D ray format) for camera pose conditioning. + +This example demonstrates: +1. Text-to-video generation with camera control +2. Using CameraNet for processing camera states +3. Integration with the FastVideo pipeline + +Usage: + python basic_hunyuangamecraft.py --prompt "A game scene" --output video_output.mp4 + +Note: This requires converted HunyuanGameCraft weights. If using official weights, +run the conversion script first: + python scripts/checkpoint_conversion/convert_gamecraft_to_fastvideo.py \ + --source /path/to/official_weights/gamecraft_model_states.pt \ + --output converted_weights/HunyuanGameCraft +""" + +import argparse +import time + +import numpy as np +import torch + +from fastvideo import VideoGenerator +from fastvideo.logger import init_logger + +logger = init_logger(__name__) + + +def create_camera_trajectory( + num_frames: int, + height: int, + width: int, + trajectory_type: str = "forward", +) -> torch.Tensor: + """ + Create camera trajectory using Plücker coordinates. + + Plücker coordinates represent a 3D line (ray) using 6 numbers: + - First 3: Direction vector (d) + - Last 3: Moment vector (m = p × d, where p is a point on the line) + + Args: + num_frames: Number of frames in the video + height: Height of the video in pixels + width: Width of the video in pixels + trajectory_type: Type of camera trajectory + - "forward": Camera moves forward + - "backward": Camera moves backward + - "left": Camera moves left + - "right": Camera moves right + - "stationary": Camera stays still + - "orbit": Camera orbits around center + + Returns: + Camera states tensor of shape [num_frames, 6, height, width] + """ + # Create a grid of pixel coordinates + y_coords = torch.linspace(-1, 1, height) + x_coords = torch.linspace(-1, 1, width) + grid_y, grid_x = torch.meshgrid(y_coords, x_coords, indexing='ij') + + camera_states = torch.zeros(num_frames, 6, height, width) + + for frame_idx in range(num_frames): + t = frame_idx / max(num_frames - 1, 1) # Normalized time [0, 1] + + # Define camera movement based on trajectory type + if trajectory_type == "forward": + # Camera moves forward (positive z direction) + direction = torch.tensor([0.0, 0.0, 1.0]) + position = torch.tensor([0.0, 0.0, -3.0 + t * 2.0]) + elif trajectory_type == "backward": + # Camera moves backward (negative z direction) + direction = torch.tensor([0.0, 0.0, -1.0]) + position = torch.tensor([0.0, 0.0, -1.0 - t * 2.0]) + elif trajectory_type == "left": + # Camera moves left (negative x direction) + direction = torch.tensor([-1.0, 0.0, 0.0]) + position = torch.tensor([1.0 - t * 2.0, 0.0, -2.0]) + elif trajectory_type == "right": + # Camera moves right (positive x direction) + direction = torch.tensor([1.0, 0.0, 0.0]) + position = torch.tensor([-1.0 + t * 2.0, 0.0, -2.0]) + elif trajectory_type == "orbit": + # Camera orbits around center + angle = t * 2 * np.pi # Full rotation + radius = 3.0 + position = torch.tensor([ + radius * np.sin(angle), + 0.0, + -radius * np.cos(angle) + ]) + # Direction points toward center + direction = -position / position.norm() + else: # stationary + direction = torch.tensor([0.0, 0.0, 1.0]) + position = torch.tensor([0.0, 0.0, -3.0]) + + # Normalize direction + direction = direction / direction.norm() + + # Compute Plücker coordinates for each pixel + # For each pixel, we create a ray from the camera through that pixel + for y in range(height): + for x in range(width): + # Pixel direction in camera space (normalized device coordinates) + pixel_dir = torch.tensor([ + grid_x[y, x].item(), + grid_y[y, x].item(), + 1.0 + ]) + pixel_dir = pixel_dir / pixel_dir.norm() + + # Transform to world space (simplified - assuming identity rotation) + world_dir = pixel_dir + + # Moment vector: m = p × d + moment = torch.cross(position, world_dir) + + # Store Plücker coordinates: [d, m] + camera_states[frame_idx, 0:3, y, x] = world_dir + camera_states[frame_idx, 3:6, y, x] = moment + + return camera_states + + +def create_simple_camera_states( + num_frames: int, + height: int, + width: int, + trajectory_type: str = "forward", +) -> torch.Tensor: + """ + Create simplified camera states for testing. + + This creates camera states using a simpler approach that's more efficient + for testing purposes. For production use, you may want to use more + sophisticated camera trajectory generation. + + Args: + num_frames: Number of frames + height: Video height + width: Video width + trajectory_type: Type of camera movement + + Returns: + Camera states tensor of shape [num_frames, 6, height, width] + """ + camera_states = torch.zeros(num_frames, 6, height, width) + + # Movement vectors for different trajectories + movement_configs = { + "forward": (0.0, 0.0, 0.1), # +z direction + "backward": (0.0, 0.0, -0.1), # -z direction + "left": (-0.1, 0.0, 0.0), # -x direction + "right": (0.1, 0.0, 0.0), # +x direction + "up": (0.0, 0.1, 0.0), # +y direction + "down": (0.0, -0.1, 0.0), # -y direction + "stationary": (0.0, 0.0, 0.0), # no movement + } + + move = movement_configs.get(trajectory_type, (0.0, 0.0, 0.0)) + + for frame_idx in range(num_frames): + t = frame_idx / max(num_frames - 1, 1) + + # Direction (first 3 channels) - forward direction with slight variation + camera_states[frame_idx, 0, :, :] = move[0] * t # x component + camera_states[frame_idx, 1, :, :] = move[1] * t # y component + camera_states[frame_idx, 2, :, :] = 1.0 + move[2] * t # z component + + # Moment (last 3 channels) - simplified + camera_states[frame_idx, 3, :, :] = 0.0 + camera_states[frame_idx, 4, :, :] = 0.0 + camera_states[frame_idx, 5, :, :] = 0.0 + + return camera_states + + +# Default settings +DEFAULT_MODEL_PATH = "converted_weights/HunyuanGameCraft" +DEFAULT_PROMPT = ( + "A first-person view of walking through a dense forest. " + "Sunlight filters through the canopy, creating dappled shadows on the ground. " + "The camera moves forward steadily." +) +DEFAULT_OUTPUT_PATH = "video_samples_hunyuangamecraft" + + +def main(): + parser = argparse.ArgumentParser( + description="HunyuanGameCraft video generation with FastVideo" + ) + parser.add_argument( + "--model-path", + type=str, + default=DEFAULT_MODEL_PATH, + help="Path to converted HunyuanGameCraft weights" + ) + parser.add_argument( + "--prompt", + type=str, + default=DEFAULT_PROMPT, + help="Text prompt for video generation" + ) + parser.add_argument( + "--negative-prompt", + type=str, + default="", + help="Negative prompt" + ) + parser.add_argument( + "--output-path", + type=str, + default=DEFAULT_OUTPUT_PATH, + help="Output video path" + ) + parser.add_argument( + "--num-frames", + type=int, + default=33, + help="Number of frames (default: 33, ~1.3s at 24fps)" + ) + parser.add_argument( + "--height", + type=int, + default=704, + help="Video height in pixels" + ) + parser.add_argument( + "--width", + type=int, + default=1280, + help="Video width in pixels" + ) + parser.add_argument( + "--num-inference-steps", + type=int, + default=50, + help="Number of denoising steps" + ) + parser.add_argument( + "--guidance-scale", + type=float, + default=6.0, + help="Classifier-free guidance scale" + ) + parser.add_argument( + "--seed", + type=int, + default=42, + help="Random seed for reproducibility" + ) + parser.add_argument( + "--trajectory", + type=str, + default="forward", + choices=["forward", "backward", "left", "right", "up", "down", "stationary"], + help="Camera trajectory type" + ) + parser.add_argument( + "--num-gpus", + type=int, + default=1, + help="Number of GPUs to use" + ) + parser.add_argument( + "--cpu-offload", + action="store_true", + help="Enable CPU offloading for memory-constrained setups" + ) + + args = parser.parse_args() + + print("=" * 60) + print("HunyuanGameCraft Video Generation") + print("=" * 60) + print(f"Model path: {args.model_path}") + print(f"Prompt: {args.prompt[:100]}...") + print(f"Resolution: {args.height}x{args.width}") + print(f"Frames: {args.num_frames}") + print(f"Camera trajectory: {args.trajectory}") + print(f"Output: {args.output_path}") + print("=" * 60) + + # Initialize generator + print("\nInitializing VideoGenerator...") + generator = VideoGenerator.from_pretrained( + args.model_path, + num_gpus=args.num_gpus, + use_fsdp_inference=args.num_gpus > 1, + dit_cpu_offload=args.cpu_offload, + vae_cpu_offload=args.cpu_offload, + text_encoder_cpu_offload=args.cpu_offload, + pin_cpu_memory=True, + ) + + # Create camera trajectory + print(f"\nCreating camera trajectory ({args.trajectory})...") + camera_states = create_simple_camera_states( + num_frames=args.num_frames, + height=args.height, + width=args.width, + trajectory_type=args.trajectory, + ) + # Add batch dimension + camera_states = camera_states.unsqueeze(0) # [1, F, 6, H, W] + + print(f"Camera states shape: {camera_states.shape}") + + # Generate video + print("\nGenerating video...") + start_time = time.time() + + result = generator.generate_video( + prompt=args.prompt, + negative_prompt=args.negative_prompt, + camera_states=camera_states, + num_frames=args.num_frames, + height=args.height, + width=args.width, + num_inference_steps=args.num_inference_steps, + guidance_scale=args.guidance_scale, + seed=args.seed, + output_path=args.output_path, + save_video=True, + ) + + elapsed = time.time() - start_time + + print(f"\nVideo generated successfully!") + print(f"Saved to: {args.output_path}") + print(f"Generation time: {elapsed:.2f}s") + print(f"Frames per second: {args.num_frames / elapsed:.2f}") + + return result + + +if __name__ == "__main__": + main() diff --git a/examples/inference/basic/basic_hunyuangamecraft_i2v.py b/examples/inference/basic/basic_hunyuangamecraft_i2v.py new file mode 100644 index 000000000..2d2b3ebe7 --- /dev/null +++ b/examples/inference/basic/basic_hunyuangamecraft_i2v.py @@ -0,0 +1,473 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Image-to-Video example for HunyuanGameCraft using FastVideo. + +HunyuanGameCraft is designed for video continuation with reference frames. +This example shows how to generate videos from a reference image with +camera control using Plücker coordinates. + +Usage: + python basic_hunyuangamecraft_i2v.py \ + --image /path/to/reference.jpg \ + --prompt "A game scene with camera moving forward" \ + --output video_output.mp4 + +Note: This requires converted HunyuanGameCraft weights. +""" + +from __future__ import annotations + +import argparse +import os +import time + +import numpy as np +import torch +from PIL import Image + +from fastvideo import VideoGenerator +from fastvideo.logger import init_logger + +logger = init_logger(__name__) + + +def custom_meshgrid(*args): + """PyTorch meshgrid with ij indexing.""" + return torch.meshgrid(*args, indexing='ij') + + +def ray_condition(K, c2w, H, W, device): + """ + Compute Plücker coordinates for camera rays. + + Args: + K: Camera intrinsics [B, V, 4] - (fx, fy, cx, cy) normalized + c2w: Camera-to-world matrices [B, V, 4, 4] + H: Height + W: Width + device: Device + + Returns: + Plücker coordinates [B, V, H, W, 6] + """ + B, V = K.shape[:2] + + j, i = custom_meshgrid( + torch.linspace(0, H - 1, H, device=device, dtype=c2w.dtype), + torch.linspace(0, W - 1, W, device=device, dtype=c2w.dtype), + ) + i = i.reshape([1, 1, H * W]).expand([B, V, H * W]) + 0.5 + j = j.reshape([1, 1, H * W]).expand([B, V, H * W]) + 0.5 + + fx, fy, cx, cy = K.chunk(4, dim=-1) # B, V, 1 + + zs = torch.ones_like(i) + xs = (i - cx) / fx * zs + ys = (j - cy) / fy * zs + zs = zs.expand_as(ys) + + directions = torch.stack((xs, ys, zs), dim=-1) # B, V, HW, 3 + directions = directions / directions.norm(dim=-1, keepdim=True) + + rays_d = directions @ c2w[..., :3, :3].transpose(-1, -2) # B, V, HW, 3 + rays_o = c2w[..., :3, 3] # B, V, 3 + rays_o = rays_o[:, :, None].expand_as(rays_d) # B, V, HW, 3 + + rays_dxo = torch.cross(rays_o, rays_d) # B, V, HW, 3 + plucker = torch.cat([rays_dxo, rays_d], dim=-1) + plucker = plucker.reshape(B, V, H, W, 6) # B, V, H, W, 6 + return plucker + + +def generate_motion_segment(current_pose, motion_type, value, duration=33): + """Generate camera motion for a segment.""" + import math + positions = [] + rotations = [] + + if motion_type in ['forward', 'backward']: + yaw_rad = np.radians(current_pose['rotation'][1]) + pitch_rad = np.radians(current_pose['rotation'][0]) + + forward_vec = np.array([ + -math.sin(yaw_rad) * math.cos(pitch_rad), + math.sin(pitch_rad), + -math.cos(yaw_rad) * math.cos(pitch_rad) + ]) + + direction = 1 if motion_type == 'forward' else -1 + total_move = forward_vec * value * direction + step = total_move / duration + + for i in range(1, duration + 1): + new_pos = current_pose['position'] + step * i + positions.append(new_pos.copy()) + rotations.append(current_pose['rotation'].copy()) + + current_pose['position'] = positions[-1] + + elif motion_type in ['left', 'right']: + import math + yaw_rad = np.radians(current_pose['rotation'][1]) + right_vec = np.array([math.cos(yaw_rad), 0, -math.sin(yaw_rad)]) + + direction = -1 if motion_type == 'right' else 1 + total_move = right_vec * value * direction + step = total_move / duration + + for i in range(1, duration + 1): + new_pos = current_pose['position'] + step * i + positions.append(new_pos.copy()) + rotations.append(current_pose['rotation'].copy()) + + current_pose['position'] = positions[-1] + else: + # Stationary - no movement + for i in range(duration): + positions.append(current_pose['position'].copy()) + rotations.append(current_pose['rotation'].copy()) + + return positions, rotations, current_pose + + +def euler_to_quaternion(angles): + """Convert euler angles to quaternion.""" + import math + pitch, yaw, roll = np.radians(angles) + + cy = math.cos(yaw * 0.5) + sy = math.sin(yaw * 0.5) + cp = math.cos(pitch * 0.5) + sp = math.sin(pitch * 0.5) + cr = math.cos(roll * 0.5) + sr = math.sin(roll * 0.5) + + qw = cy * cp * cr + sy * sp * sr + qx = cy * cp * sr - sy * sp * cr + qy = sy * cp * sr + cy * sp * cr + qz = sy * cp * cr - cy * sp * sr + + return [qw, qx, qy, qz] + + +def quaternion_to_rotation_matrix(q): + """Convert quaternion to rotation matrix.""" + qw, qx, qy, qz = q + return np.array([ + [1 - 2*(qy**2 + qz**2), 2*(qx*qy - qw*qz), 2*(qx*qz + qw*qy)], + [2*(qx*qy + qw*qz), 1 - 2*(qx**2 + qz**2), 2*(qy*qz - qw*qx)], + [2*(qx*qz - qw*qy), 2*(qy*qz + qw*qx), 1 - 2*(qx**2 + qy**2)] + ]) + + +def create_camera_states( + num_frames: int, + height: int, + width: int, + trajectory_type: str = "forward", + action_speed: float = 0.2, +) -> torch.Tensor: + """ + Create proper Plücker coordinate camera states matching official implementation. + + Args: + num_frames: Number of frames + height: Video height + width: Video width + trajectory_type: Type of camera movement + action_speed: Speed of movement + + Returns: + Camera states tensor of shape [1, num_frames, 6, height, width] + """ + ACTION_DICT = { + "forward": "forward", + "backward": "backward", + "left": "left", + "right": "right", + "stationary": "stationary", + "up": "forward", # Map up/down to forward for now + "down": "backward", + } + + motion_type = ACTION_DICT.get(trajectory_type, "forward") + + # Generate motion poses + current_pose = { + 'position': np.array([0.0, 0.0, 0.0]), + 'rotation': np.array([0.0, 0.0, 0.0]) + } + + positions, rotations, _ = generate_motion_segment( + current_pose, motion_type, action_speed, num_frames + ) + + # Camera intrinsics (normalized) + intrinsic = [0.50505, 0.8979, 0.5, 0.5] # fx, fy, cx, cy + fx_norm = intrinsic[0] * width + fy_norm = intrinsic[1] * height + cx_norm = intrinsic[2] * width + cy_norm = intrinsic[3] * height + + # Build c2w matrices from poses + c2w_list = [] + + # First frame is identity + c2w_first = np.eye(4, dtype=np.float32) + c2w_list.append(c2w_first) + + # Remaining frames based on motion + for pos, rot in zip(positions, rotations): + quat = euler_to_quaternion(rot) + R = quaternion_to_rotation_matrix(quat) + c2w = np.eye(4, dtype=np.float32) + c2w[:3, :3] = R + c2w[:3, 3] = pos * 2 # Scale position like official code + c2w_list.append(c2w) + + # Only take num_frames + c2w_list = c2w_list[:num_frames] + + # Pad if needed + while len(c2w_list) < num_frames: + c2w_list.append(c2w_list[-1]) + + c2w = torch.tensor(np.array(c2w_list, dtype=np.float32))[None] # [1, F, 4, 4] + + # Camera intrinsics for all frames + K = torch.tensor([[[fx_norm, fy_norm, cx_norm, cy_norm]]], dtype=torch.float32) + K = K.expand(1, num_frames, 4) # [1, F, 4] + + # Compute Plücker embeddings + plucker = ray_condition(K, c2w, height, width, device='cpu') # [1, F, H, W, 6] + plucker = plucker[0].permute(0, 3, 1, 2) # [F, 6, H, W] + + return plucker.unsqueeze(0) # [1, F, 6, H, W] + + +def create_simple_camera_states( + num_frames: int, + height: int, + width: int, + trajectory_type: str = "forward", +) -> torch.Tensor: + """ + Wrapper for backwards compatibility. + """ + return create_camera_states(num_frames, height, width, trajectory_type, action_speed=0.2) + + +def load_and_preprocess_image( + image_path: str, + target_height: int, + target_width: int, +) -> Image.Image: + """Load and resize image to target dimensions.""" + image = Image.open(image_path).convert("RGB") + image = image.resize((target_width, target_height), Image.LANCZOS) + return image + + +def create_sample_image(height: int, width: int) -> Image.Image: + """Create a sample gradient image for testing when no image is provided.""" + # Create a simple gradient image + img_array = np.zeros((height, width, 3), dtype=np.uint8) + + # Create a sky-to-ground gradient (blue to green) + for y in range(height): + t = y / height + # Sky (blue) to ground (green/brown) + r = int(100 * t + 50 * (1 - t)) + g = int(150 * t + 100 * (1 - t)) + b = int(50 * t + 200 * (1 - t)) + img_array[y, :, 0] = r + img_array[y, :, 1] = g + img_array[y, :, 2] = b + + return Image.fromarray(img_array) + + +# Default settings +DEFAULT_MODEL_PATH = "converted_weights/HunyuanGameCraft" +DEFAULT_PROMPT = ( + "A first-person view moving through a forest environment. " + "Trees and foliage visible on both sides. " + "Natural lighting with shadows." +) +DEFAULT_OUTPUT_PATH = "video_samples_hunyuangamecraft_i2v" + + +def main(): + parser = argparse.ArgumentParser( + description="HunyuanGameCraft Image-to-Video generation with FastVideo" + ) + parser.add_argument( + "--model-path", + type=str, + default=DEFAULT_MODEL_PATH, + help="Path to converted HunyuanGameCraft weights" + ) + parser.add_argument( + "--image", + type=str, + default=None, + help="Path to reference image (if not provided, uses a sample gradient)" + ) + parser.add_argument( + "--prompt", + type=str, + default=DEFAULT_PROMPT, + help="Text prompt for video generation" + ) + parser.add_argument( + "--negative-prompt", + type=str, + default="", + help="Negative prompt" + ) + parser.add_argument( + "--output-path", + type=str, + default=DEFAULT_OUTPUT_PATH, + help="Output directory" + ) + parser.add_argument( + "--num-frames", + type=int, + default=37, + help="Number of frames (default: 37; gives 10 latent frames via (37-1)//4+1=10, matching official's 10 latent frames)" + ) + parser.add_argument( + "--height", + type=int, + default=480, + help="Video height in pixels" + ) + parser.add_argument( + "--width", + type=int, + default=720, + help="Video width in pixels" + ) + parser.add_argument( + "--num-inference-steps", + type=int, + default=50, + help="Number of denoising steps" + ) + parser.add_argument( + "--guidance-scale", + type=float, + default=6.0, + help="Classifier-free guidance scale (official default is 6.0 with flow_shift=5.0)" + ) + parser.add_argument( + "--seed", + type=int, + default=42, + help="Random seed for reproducibility" + ) + parser.add_argument( + "--trajectory", + type=str, + default="forward", + choices=["forward", "backward", "left", "right", "up", "down", "stationary"], + help="Camera trajectory type" + ) + parser.add_argument( + "--num-gpus", + type=int, + default=1, + help="Number of GPUs to use" + ) + parser.add_argument( + "--cpu-offload", + action="store_true", + help="Enable CPU offloading for memory-constrained setups" + ) + + args = parser.parse_args() + + print("=" * 60) + print("HunyuanGameCraft Image-to-Video Generation") + print("=" * 60) + print(f"Model path: {args.model_path}") + print(f"Reference image: {args.image or 'Using sample gradient'}") + print(f"Prompt: {args.prompt[:80]}...") + print(f"Resolution: {args.height}x{args.width}") + print(f"Frames: {args.num_frames}") + print(f"Camera trajectory: {args.trajectory}") + print(f"Output: {args.output_path}") + print("=" * 60) + + # Load or create reference image + if args.image and os.path.exists(args.image): + print(f"\nLoading reference image from {args.image}...") + ref_image = load_and_preprocess_image(args.image, args.height, args.width) + else: + print("\nNo reference image provided, creating sample gradient...") + ref_image = create_sample_image(args.height, args.width) + + # Initialize generator + print("\nInitializing VideoGenerator...") + generator = VideoGenerator.from_pretrained( + args.model_path, + num_gpus=args.num_gpus, + use_fsdp_inference=args.num_gpus > 1, + dit_cpu_offload=args.cpu_offload, + vae_cpu_offload=args.cpu_offload, + text_encoder_cpu_offload=args.cpu_offload, + pin_cpu_memory=True, + ) + + # Create camera trajectory. + # The official implementation generates 33 camera poses (one per generated + # frame, excluding the reference frame). The CameraNet handles the + # temporal mapping from 33 camera frames to latent frame tokens internally. + camera_num_frames = 33 # Always 33 camera poses for single-image I2V + print(f"\nCreating camera trajectory ({args.trajectory}, {camera_num_frames} camera frames)...") + camera_states = create_simple_camera_states( + num_frames=camera_num_frames, + height=args.height, + width=args.width, + trajectory_type=args.trajectory, + ) + # create_simple_camera_states already returns [1, F, 6, H, W] + # No additional unsqueeze needed + + print(f"Camera states shape: {camera_states.shape}") + + # Create output directory + os.makedirs(args.output_path, exist_ok=True) + + # Generate video + print("\nGenerating video...") + start_time = time.time() + + result = generator.generate_video( + prompt=args.prompt, + negative_prompt=args.negative_prompt, + pil_image=ref_image, # Pass reference image (field name is pil_image in SamplingParam) + camera_states=camera_states, + num_frames=args.num_frames, + height=args.height, + width=args.width, + num_inference_steps=args.num_inference_steps, + guidance_scale=args.guidance_scale, + seed=args.seed, + output_path=args.output_path, + save_video=True, + ) + + elapsed = time.time() - start_time + + print(f"\nVideo generated successfully!") + print(f"Saved to: {args.output_path}") + print(f"Generation time: {elapsed:.2f}s") + print(f"Frames per second: {args.num_frames / elapsed:.2f}") + + return result + + +if __name__ == "__main__": + main() diff --git a/fastvideo/configs/models/dits/hunyuangamecraft.py b/fastvideo/configs/models/dits/hunyuangamecraft.py index 33ef36a88..a9ece0290 100644 --- a/fastvideo/configs/models/dits/hunyuangamecraft.py +++ b/fastvideo/configs/models/dits/hunyuangamecraft.py @@ -183,10 +183,12 @@ class HunyuanGameCraftArchConfig(HunyuanVideoArchConfig): camera_in_channels: int = 6 # Plücker coordinates: 6D camera_downscale_coef: int = 8 - # HunyuanVideo architecture defaults (same as base HunyuanVideo) + # HunyuanVideo architecture defaults + # Note: in_channels=33 because official uses multitask_mask_training_type="concat" + # which expands 16 latent channels to 16*2+1=33 (latent + conditioning + mask) patch_size: int = 2 patch_size_t: int = 1 - in_channels: int = 16 + in_channels: int = 33 # 16 latent + 16 cond + 1 mask out_channels: int = 16 num_attention_heads: int = 24 attention_head_dim: int = 128 @@ -195,7 +197,7 @@ class HunyuanGameCraftArchConfig(HunyuanVideoArchConfig): num_single_layers: int = 40 # single stream blocks num_refiner_layers: int = 2 rope_axes_dim: tuple[int, int, int] = (16, 56, 56) - guidance_embeds: bool = True # GameCraft uses guidance embeddings + guidance_embeds: bool = False # Official GameCraft checkpoint doesn't have guidance_in dtype: torch.dtype | None = None text_embed_dim: int = 4096 pooled_projection_dim: int = 768 @@ -204,11 +206,16 @@ class HunyuanGameCraftArchConfig(HunyuanVideoArchConfig): exclude_lora_layers: list[str] = field( default_factory=lambda: ["img_in", "txt_in", "time_in", "vector_in", "camera_net"]) + + prefix: str = "HunyuanGameCraft" def __post_init__(self): super().__post_init__() self.hidden_size: int = self.attention_head_dim * self.num_attention_heads - self.num_channels_latents: int = self.in_channels + # num_channels_latents is the VAE output channels (16), not in_channels (33) + # in_channels = 16 (latent) + 16 (conditioning) + 1 (mask) = 33 + # out_channels = 16 (latent channels only) + self.num_channels_latents: int = self.out_channels @dataclass