Add LongCat-Video I2V and Video Continuation (Base, Distillation and Refinement) Support to FastVideo (#953)
This commit is contained in:
Shao Duan
@@ -1,14 +1,30 @@
|
||||
#!/bin/bash
|
||||
|
||||
num_gpus=2
|
||||
# LongCat Text-to-Video (T2V) Inference Script
|
||||
#
|
||||
# This script runs LongCat T2V inference using the fastvideo CLI.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/inference/v1_inference_longcat.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Install fastvideo: pip install -e .
|
||||
# - The model weights will be auto-downloaded from HuggingFace
|
||||
|
||||
num_gpus=1
|
||||
|
||||
export FASTVIDEO_ATTENTION_BACKEND=
|
||||
|
||||
# For longcat, we must first convert the official weights to FastVideo native format
|
||||
# Model path options:
|
||||
# Option 1: HuggingFace model (auto-downloaded)
|
||||
export MODEL_BASE=FastVideo/LongCat-Video-T2V-Diffusers
|
||||
|
||||
# Option 2: Local weights (uncomment if you have local weights)
|
||||
# For local weights, convert the official weights to FastVideo native format
|
||||
# conversion method: python scripts/checkpoint_conversion/longcat_to_fastvideo.py
|
||||
# --source /path/to/LongCat-Video/weights/LongCat-Video
|
||||
# --output weights/longcat-native
|
||||
export MODEL_BASE=weights/longcat-native
|
||||
# export MODEL_BASE=weights/longcat-native
|
||||
|
||||
fastvideo generate \
|
||||
--model-path $MODEL_BASE \
|
||||
@@ -26,7 +42,7 @@ fastvideo generate \
|
||||
--num-inference-steps 50 \
|
||||
--fps 15 \
|
||||
--guidance-scale 4.0 \
|
||||
--prompt-txt assets/prompt.txt \
|
||||
--prompt "In a realistic photography style, a white boy around seven or eight years old sits on a park bench, wearing a light blue T-shirt, denim shorts, and white sneakers. He holds an ice cream cone with vanilla and chocolate flavors, and beside him is a medium-sized golden Labrador. Smiling, the boy offers the ice cream to the dog, who eagerly licks it with its tongue. The sun is shining brightly, and the background features a green lawn and several tall trees, creating a warm and loving scene." \
|
||||
--negative-prompt "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" \
|
||||
--seed 42 \
|
||||
--output-path outputs_video/longcat_480p
|
||||
--output-path outputs_video/longcat_t2v
|
||||
|
||||
@@ -1,12 +1,23 @@
|
||||
#!/bin/bash
|
||||
|
||||
# LongCat T2V Distilled Inference Script
|
||||
#
|
||||
# This script runs LongCat T2V with distillation LoRA (16 steps instead of 50).
|
||||
# Uses the distilled LoRA for faster generation.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/inference/v1_inference_longcat_distill.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Install fastvideo: pip install -e .
|
||||
# - The model weights will be auto-downloaded from HuggingFace
|
||||
|
||||
num_gpus=1
|
||||
|
||||
export FASTVIDEO_ATTENTION_BACKEND=
|
||||
# For longcat, we must first convert the official weights to FastVideo native format
|
||||
# conversion method: python scripts/checkpoint_conversion/longcat_to_fastvideo.py
|
||||
# --source /path/to/LongCat-Video/weights/LongCat-Video
|
||||
# --output weights/longcat-native
|
||||
export MODEL_BASE=weights/longcat-native
|
||||
|
||||
# Model path - HuggingFace model (auto-downloaded)
|
||||
export MODEL_BASE=FastVideo/LongCat-Video-T2V-Diffusers
|
||||
|
||||
fastvideo generate \
|
||||
--model-path $MODEL_BASE \
|
||||
@@ -14,11 +25,11 @@ fastvideo generate \
|
||||
--tp-size 1 \
|
||||
--num-gpus $num_gpus \
|
||||
--dit-cpu-offload False \
|
||||
--vae-cpu-offload False \
|
||||
--text-encoder-cpu-offload False \
|
||||
--vae-cpu-offload True \
|
||||
--text-encoder-cpu-offload True \
|
||||
--pin-cpu-memory False \
|
||||
--enable-bsa False \
|
||||
--lora-path "$MODEL_BASE/lora/distilled" \
|
||||
--lora-path "FastVideo/LongCat-Video-T2V-Distilled-LoRA" \
|
||||
--lora-nickname "distilled" \
|
||||
--height 480 \
|
||||
--width 832 \
|
||||
@@ -26,6 +37,7 @@ fastvideo generate \
|
||||
--num-inference-steps 16 \
|
||||
--fps 15 \
|
||||
--guidance-scale 1.0 \
|
||||
--prompt "In a realistic photography style, an asian boy around seven or eight years old sits on a park bench, wearing a light yellow T-shirt, denim shorts, and white sneakers. He holds an ice cream cone with vanilla and chocolate flavors, and beside him is a medium-sized golden Labrador. Smiling, the boy offers the ice cream to the dog, who eagerly licks it with its tongue. The sun is shining brightly, and the background features a green lawn and several tall trees, creating a warm and loving scene." \
|
||||
--prompt "In a realistic photography style, a white boy around seven or eight years old sits on a park bench, wearing a light blue T-shirt, denim shorts, and white sneakers. He holds an ice cream cone with vanilla and chocolate flavors, and beside him is a medium-sized golden Labrador. Smiling, the boy offers the ice cream to the dog, who eagerly licks it with its tongue. The sun is shining brightly, and the background features a green lawn and several tall trees, creating a warm and loving scene." \
|
||||
--negative-prompt "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" \
|
||||
--seed 42 \
|
||||
--output-path outputs_video/longcat_distill
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/bin/bash
|
||||
|
||||
# LongCat Image-to-Video (I2V) Inference Script
|
||||
#
|
||||
# This script runs LongCat I2V inference using the fastvideo CLI.
|
||||
# LongCat I2V takes an input image and generates a video from it.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/inference/v1_inference_longcat_i2v.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Install fastvideo: pip install -e .
|
||||
# - The model weights will be auto-downloaded from HuggingFace
|
||||
# - Or use local weights if you have them
|
||||
|
||||
num_gpus=1
|
||||
|
||||
export FASTVIDEO_ATTENTION_BACKEND=
|
||||
|
||||
# Model path options:
|
||||
# Option 1: HuggingFace model (auto-downloaded)
|
||||
export MODEL_BASE=FastVideo/LongCat-Video-I2V-Diffusers
|
||||
|
||||
# Option 2: Local weights (uncomment if you have local weights)
|
||||
# export MODEL_BASE=weights/longcat-for-i2v
|
||||
|
||||
# Input image path (must be square for LongCat I2V)
|
||||
IMAGE_PATH="assets/girl.png"
|
||||
|
||||
# Check if image exists
|
||||
if [ ! -f "$IMAGE_PATH" ]; then
|
||||
echo "Error: Image not found at $IMAGE_PATH"
|
||||
echo "Please provide a valid image path"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
fastvideo generate \
|
||||
--model-path $MODEL_BASE \
|
||||
--sp-size $num_gpus \
|
||||
--tp-size 1 \
|
||||
--num-gpus $num_gpus \
|
||||
--dit-cpu-offload False \
|
||||
--vae-cpu-offload True \
|
||||
--text-encoder-cpu-offload True \
|
||||
--pin-cpu-memory False \
|
||||
--enable-bsa False \
|
||||
--image-path "$IMAGE_PATH" \
|
||||
--height 480 \
|
||||
--width 480 \
|
||||
--num-frames 93 \
|
||||
--num-inference-steps 50 \
|
||||
--fps 15 \
|
||||
--guidance-scale 4.0 \
|
||||
--prompt "A woman sits at a wooden table by the window in a cozy café. She reaches out with her right hand, picks up the white coffee cup from the saucer, and gently brings it to her lips to take a sip. After drinking, she places the cup back on the table and looks out the window, enjoying the peaceful atmosphere." \
|
||||
--negative-prompt "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" \
|
||||
--seed 42 \
|
||||
--output-path outputs_video/longcat_i2v
|
||||
|
||||
|
||||
|
||||
@@ -1,18 +1,30 @@
|
||||
#!/bin/bash
|
||||
|
||||
num_gpus=1
|
||||
export FASTVIDEO_ATTENTION_BACKEND=
|
||||
# For longcat, we must first convert the official weights to FastVideo native format
|
||||
# conversion method: python scripts/checkpoint_conversion/longcat_to_fastvideo.py
|
||||
# --source /path/to/LongCat-Video/weights/LongCat-Video
|
||||
# --output weights/longcat-native
|
||||
export MODEL_BASE=weights/longcat-native
|
||||
# LongCat T2V Refinement Script (480p -> 720p)
|
||||
#
|
||||
# This script refines a 480p distilled video to 720p using the refinement LoRA.
|
||||
# Run v1_inference_longcat_distill.sh first to generate the 480p video.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/inference/v1_inference_longcat_refine_fromvideo.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Install fastvideo: pip install -e .
|
||||
# - The model weights will be auto-downloaded from HuggingFace
|
||||
# - Run v1_inference_longcat_distill.sh first to generate input video
|
||||
|
||||
INPUT_VIDEO="outputs_video/longcat_distill/In a realistic photography style, an asian boy around seven or eight years old sits on a park bench,.mp4"
|
||||
num_gpus=1
|
||||
|
||||
export FASTVIDEO_ATTENTION_BACKEND=
|
||||
|
||||
# Model path - HuggingFace model (auto-downloaded)
|
||||
export MODEL_BASE=FastVideo/LongCat-Video-T2V-Diffusers
|
||||
|
||||
INPUT_VIDEO="outputs_video/longcat_distill/In a realistic photography style, a white boy around seven or eight years old sits on a park bench,.mp4"
|
||||
REFINE_OUTPUT="outputs_video/longcat_refine_720p"
|
||||
|
||||
# Prompt used for base generation
|
||||
PROMPT="In a realistic photography style, an asian boy around seven or eight years old sits on a park bench, wearing a light yellow T-shirt, denim shorts, and white sneakers. He holds an ice cream cone with vanilla and chocolate flavors, and beside him is a medium-sized golden Labrador. Smiling, the boy offers the ice cream to the dog, who eagerly licks it with its tongue. The sun is shining brightly, and the background features a green lawn and several tall trees, creating a warm and loving scene."
|
||||
# Prompt used for base generation (must match distill script)
|
||||
PROMPT="In a realistic photography style, a white boy around seven or eight years old sits on a park bench, wearing a light blue T-shirt, denim shorts, and white sneakers. He holds an ice cream cone with vanilla and chocolate flavors, and beside him is a medium-sized golden Labrador. Smiling, the boy offers the ice cream to the dog, who eagerly licks it with its tongue. The sun is shining brightly, and the background features a green lawn and several tall trees, creating a warm and loving scene."
|
||||
|
||||
echo "=========================================="
|
||||
echo "LongCat 480p -> 720p Refinement"
|
||||
@@ -25,14 +37,14 @@ echo ""
|
||||
# Check if input video exists
|
||||
if [ ! -f "$INPUT_VIDEO" ]; then
|
||||
echo "Error: Input video not found: $INPUT_VIDEO"
|
||||
echo "Please set INPUT_VIDEO to your 480p video path"
|
||||
echo "Please run v1_inference_longcat_distill.sh first to generate the 480p video"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "🔧 Configuring refinement (BSA enabled, refinement LoRA)..."
|
||||
echo "✅ Input video: $INPUT_VIDEO"
|
||||
echo "✅ BSA enabled with sparsity=0.875"
|
||||
echo "✅ Refinement LoRA loaded"
|
||||
echo "Configuring refinement (BSA enabled, refinement LoRA)..."
|
||||
echo "Input video: $INPUT_VIDEO"
|
||||
echo "BSA enabled with sparsity=0.875"
|
||||
echo "Refinement LoRA loaded"
|
||||
echo ""
|
||||
|
||||
fastvideo generate \
|
||||
@@ -41,14 +53,14 @@ fastvideo generate \
|
||||
--tp-size 1 \
|
||||
--num-gpus $num_gpus \
|
||||
--dit-cpu-offload True \
|
||||
--vae-cpu-offload False \
|
||||
--vae-cpu-offload True \
|
||||
--text-encoder-cpu-offload True \
|
||||
--pin-cpu-memory False \
|
||||
--enable-bsa True \
|
||||
--bsa-sparsity 0.875 \
|
||||
--bsa-chunk-q 4 4 8 \
|
||||
--bsa-chunk-k 4 4 8 \
|
||||
--lora-path "$MODEL_BASE/lora/refinement" \
|
||||
--lora-path "FastVideo/LongCat-Video-T2V-Refinement-LoRA" \
|
||||
--lora-nickname "refinement" \
|
||||
--refine-from "$INPUT_VIDEO" \
|
||||
--t-thresh 0.5 \
|
||||
@@ -60,12 +72,13 @@ fastvideo generate \
|
||||
--fps 30 \
|
||||
--guidance-scale 1.0 \
|
||||
--prompt "$PROMPT" \
|
||||
--negative-prompt "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" \
|
||||
--seed 42 \
|
||||
--output-path "$REFINE_OUTPUT"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "✓ Refinement Complete!"
|
||||
echo "Refinement Complete!"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Output directory: $REFINE_OUTPUT"
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
# LongCat Video Continuation (VC) Inference Script
|
||||
#
|
||||
# This script runs LongCat VC inference using the fastvideo CLI.
|
||||
# LongCat VC takes an input video and generates a continuation of it.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/inference/v1_inference_longcat_vc.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Install fastvideo: pip install -e .
|
||||
# - The model weights will be auto-downloaded from HuggingFace
|
||||
# - Or use local weights if you have them
|
||||
|
||||
num_gpus=1
|
||||
|
||||
export FASTVIDEO_ATTENTION_BACKEND=
|
||||
|
||||
# Model path options:
|
||||
# Option 1: HuggingFace model (auto-downloaded)
|
||||
export MODEL_BASE=FastVideo/LongCat-Video-VC-Diffusers
|
||||
|
||||
# Option 2: Local weights (uncomment if you have local weights)
|
||||
# export MODEL_BASE=weights/longcat-vc-upload
|
||||
|
||||
# Input video path
|
||||
VIDEO_PATH="assets/motorcycle.mp4"
|
||||
|
||||
# Check if video exists
|
||||
if [ ! -f "$VIDEO_PATH" ]; then
|
||||
echo "Error: Video not found at $VIDEO_PATH"
|
||||
echo "Please provide a valid video path"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
fastvideo generate \
|
||||
--model-path $MODEL_BASE \
|
||||
--sp-size $num_gpus \
|
||||
--tp-size 1 \
|
||||
--num-gpus $num_gpus \
|
||||
--dit-cpu-offload False \
|
||||
--vae-cpu-offload True \
|
||||
--text-encoder-cpu-offload True \
|
||||
--pin-cpu-memory False \
|
||||
--enable-bsa False \
|
||||
--video-path "$VIDEO_PATH" \
|
||||
--num-cond-frames 13 \
|
||||
--height 480 \
|
||||
--width 832 \
|
||||
--num-frames 93 \
|
||||
--num-inference-steps 50 \
|
||||
--fps 15 \
|
||||
--guidance-scale 4.0 \
|
||||
--prompt "A person rides a motorcycle along a long, straight road that stretches between a body of water and a forested hillside. The rider steadily accelerates, keeping the motorcycle centered between the guardrails, while the scenery passes by on both sides. The video captures the journey from the rider's perspective, emphasizing the sense of motion and adventure." \
|
||||
--negative-prompt "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" \
|
||||
--seed 42 \
|
||||
--output-path outputs_video/longcat_vc
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user