Will/training (#425)
Co-authored-by: Kevin Lin <42618777+kevin314@users.noreply.github.com> Co-authored-by: William Lin <SolitaryThinker@users.noreply.github.com> Co-authored-by: Will Lin <wlsaidhi@gmail.com>
This commit is contained in:
Wei Zhou
committed by
Will Lin
Will Lin
co-authored by
Kevin Lin
William Lin
Will Lin
Kevin Lin
William Lin
Will Lin
parent
7f4379acc0
commit
d36c209191
@@ -1,11 +1,11 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import csv
|
||||
import cv2
|
||||
|
||||
|
||||
def get_video_info(video_path, prompt_text):
|
||||
"""Extract video information using OpenCV and corresponding prompt text"""
|
||||
def get_video_info(video_path, metadata):
|
||||
"""Extract video information using OpenCV and corresponding metadata"""
|
||||
cap = cv2.VideoCapture(str(video_path))
|
||||
|
||||
if not cap.isOpened():
|
||||
@@ -23,60 +23,66 @@ def get_video_info(video_path, prompt_text):
|
||||
|
||||
return {
|
||||
"path": video_path.name,
|
||||
"title": metadata.get("Video Title", ""),
|
||||
"description": metadata.get("Video Description", ""),
|
||||
"video_url": metadata.get("Video URL", ""),
|
||||
"download_url": metadata.get("Download URL", ""),
|
||||
"resolution": {
|
||||
"width": width,
|
||||
"height": height
|
||||
},
|
||||
"fps": fps,
|
||||
"duration": duration,
|
||||
"cap": [prompt_text]
|
||||
"cap": [metadata.get("Video Description", "")]
|
||||
}
|
||||
|
||||
|
||||
def read_prompt_file(prompt_path):
|
||||
"""Read and return the content of a prompt file"""
|
||||
def read_csv_file(csv_path):
|
||||
"""Read and return the content of a CSV file"""
|
||||
try:
|
||||
with open(prompt_path, 'r', encoding='utf-8') as f:
|
||||
return f.read().strip()
|
||||
with open(csv_path, 'r', encoding='utf-8') as f:
|
||||
reader = csv.DictReader(f)
|
||||
return list(reader)
|
||||
except Exception as e:
|
||||
print(f"Error reading prompt file {prompt_path}: {e}")
|
||||
print(f"Error reading CSV file {csv_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def process_videos_and_prompts(video_dir_path, prompt_dir_path, verbose=False):
|
||||
"""Process videos and their corresponding prompt files
|
||||
def process_videos_from_csv(video_dir_path, csv_path, verbose=False):
|
||||
"""Process videos using metadata from CSV file
|
||||
|
||||
Args:
|
||||
video_dir_path (str): Path to directory containing video files
|
||||
prompt_dir_path (str): Path to directory containing prompt files
|
||||
csv_path (str): Path to CSV file containing video metadata
|
||||
verbose (bool): Whether to print verbose processing information
|
||||
"""
|
||||
video_dir = Path(video_dir_path)
|
||||
prompt_dir = Path(prompt_dir_path)
|
||||
csv_data = read_csv_file(csv_path)
|
||||
processed_data = []
|
||||
|
||||
# Ensure directories exist
|
||||
if not video_dir.exists() or not prompt_dir.exists():
|
||||
print(f"Error: One or both directories do not exist:\nVideos: {video_dir}\nPrompts: {prompt_dir}")
|
||||
if not video_dir.exists():
|
||||
print(f"Error: Video directory does not exist: {video_dir}")
|
||||
return []
|
||||
|
||||
if csv_data is None:
|
||||
return []
|
||||
|
||||
# Process each video file
|
||||
for video_file in video_dir.glob('*.mp4'):
|
||||
video_name = video_file.stem
|
||||
prompt_file = prompt_dir / f"{video_name}.txt"
|
||||
|
||||
# Check if corresponding prompt file exists
|
||||
if not prompt_file.exists():
|
||||
print(f"Warning: No prompt file found for video {video_name}")
|
||||
for row in csv_data:
|
||||
video_filename = row.get("Filename")
|
||||
if not video_filename:
|
||||
continue
|
||||
|
||||
# Read prompt content
|
||||
prompt_text = read_prompt_file(prompt_file)
|
||||
if prompt_text is None:
|
||||
video_file = video_dir / video_filename
|
||||
|
||||
# Check if video file exists
|
||||
if not video_file.exists():
|
||||
print(f"Warning: Video file not found: {video_filename}")
|
||||
continue
|
||||
|
||||
# Process video and add to results
|
||||
video_info = get_video_info(video_file, prompt_text)
|
||||
video_info = get_video_info(video_file, row)
|
||||
if video_info:
|
||||
processed_data.append(video_info)
|
||||
|
||||
@@ -105,9 +111,9 @@ def parse_args():
|
||||
"""Parse command line arguments"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='Process videos and their corresponding prompt files')
|
||||
parser = argparse.ArgumentParser(description='Process videos using metadata from CSV file')
|
||||
parser.add_argument('--video_dir', '-v', required=True, help='Directory containing video files')
|
||||
parser.add_argument('--prompt_dir', '-p', required=True, help='Directory containing prompt text files')
|
||||
parser.add_argument('--csv_path', '-c', required=True, help='Path to CSV file containing video metadata')
|
||||
parser.add_argument('--output_path',
|
||||
'-o',
|
||||
required=True,
|
||||
@@ -121,8 +127,8 @@ if __name__ == "__main__":
|
||||
# Parse command line arguments
|
||||
args = parse_args()
|
||||
|
||||
# Process videos and prompts
|
||||
processed_videos = process_videos_and_prompts(args.video_dir, args.prompt_dir, args.verbose)
|
||||
# Process videos from CSV
|
||||
processed_videos = process_videos_from_csv(args.video_dir, args.csv_path, args.verbose)
|
||||
|
||||
if processed_videos:
|
||||
# Save results
|
||||
|
||||
@@ -24,9 +24,9 @@ def is_16_9_ratio(width: int, height: int, tolerance: float = 0.1) -> bool:
|
||||
def resize_video(args_tuple):
|
||||
"""
|
||||
Resize a single video file.
|
||||
args_tuple: (input_file, output_dir, width, height, fps)
|
||||
args_tuple: (input_file, output_dir, width, height, fps, num_frames)
|
||||
"""
|
||||
input_file, output_dir, width, height, fps = args_tuple
|
||||
input_file, output_dir, width, height, fps, num_frames = args_tuple
|
||||
video = None
|
||||
resized = None
|
||||
output_file = output_dir / f"{input_file.name}"
|
||||
@@ -39,6 +39,13 @@ def resize_video(args_tuple):
|
||||
if not is_16_9_ratio(video.w, video.h):
|
||||
return (input_file.name, "skipped", "Not 16:9")
|
||||
|
||||
# Calculate target duration based on num_frames and fps
|
||||
target_duration = num_frames / fps
|
||||
|
||||
# Trim video if it's longer than target duration
|
||||
if video.duration > target_duration:
|
||||
video = video.subclip(0, target_duration)
|
||||
|
||||
def process_frame(frame):
|
||||
frame_float = frame.astype(float) / 255.0
|
||||
resized = resize(frame_float, (height, width, 3), mode='reflect', anti_aliasing=True, preserve_range=True)
|
||||
@@ -75,7 +82,7 @@ def process_folder(args):
|
||||
print(f"Target: {args.width}x{args.height} at {args.fps}fps")
|
||||
|
||||
# Prepare arguments for parallel processing
|
||||
process_args = [(video_file, output_path, args.width, args.height, args.fps) for video_file in video_files]
|
||||
process_args = [(video_file, output_path, args.width, args.height, args.fps, args.num_frames) for video_file in video_files]
|
||||
|
||||
successful = 0
|
||||
skipped = 0
|
||||
@@ -115,6 +122,7 @@ def parse_args():
|
||||
parser.add_argument('--width', type=int, default=1280, help='Target width in pixels (default: 848)')
|
||||
parser.add_argument('--height', type=int, default=720, help='Target height in pixels (default: 480)')
|
||||
parser.add_argument('--fps', type=int, default=30, help='Target frames per second (default: 30)')
|
||||
parser.add_argument('--num_frames', type=int, default=163, help='Target number of frames (default: 163)')
|
||||
parser.add_argument('--max_workers',
|
||||
type=int,
|
||||
default=4,
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
num_gpus=2
|
||||
export FASTVIDEO_ATTENTION_BACKEND=
|
||||
export MODEL_BASE=Wan-AI/Wan2.1-T2V-1.3B-Diffusers
|
||||
# export MODEL_BASE=hunyuanvideo-community/HunyuanVideo
|
||||
# Note that the tp_size and sp_size should be the same and equal to the number
|
||||
# of GPUs. They are used for different parallel groups. sp_size is used for
|
||||
# dit model and tp_size is used for encoder models.
|
||||
torchrun --nnodes=1 --nproc_per_node=$num_gpus --master_port 29503 \
|
||||
fastvideo/v1/entrypoints/data_preprocessor.py \
|
||||
--sp_size $num_gpus \
|
||||
--tp_size $num_gpus \
|
||||
--height 480 \
|
||||
--width 832 \
|
||||
--num_frames 77 \
|
||||
--num_inference_steps 50 \
|
||||
--fps 16 \
|
||||
--guidance_scale 3.0 \
|
||||
--prompt_path ./assets/prompt.txt \
|
||||
--neg_prompt "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards" \
|
||||
--seed 1024 \
|
||||
--output_path outputs_video/ \
|
||||
--model_path $MODEL_BASE \
|
||||
--vae-sp \
|
||||
--text-encoder-precision "fp32" \
|
||||
--use-cpu-offload
|
||||
@@ -0,0 +1,30 @@
|
||||
# export WANDB_MODE="offline"
|
||||
GPU_NUM=1 # 2,4,8
|
||||
# MODEL_PATH="/home/ray/.cache/huggingface/hub/models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers/snapshots/0fad780a534b6463e45facd96134c9f345acfa5b"
|
||||
MODEL_PATH="Wan-AI/Wan2.1-T2V-1.3B-Diffusers"
|
||||
DATA_MERGE_PATH="data/cats_480/merge.txt"
|
||||
OUTPUT_DIR="data/cats_480_latents/"
|
||||
VALIDATION_PATH="assets/prompt.txt"
|
||||
|
||||
# torchrun --nproc_per_node=$GPU_NUM \
|
||||
# fastvideo/data_preprocess/preprocess_vae_latents_v1.py \
|
||||
# --model_path $MODEL_PATH \
|
||||
# --data_merge_path $DATA_MERGE_PATH \
|
||||
# --train_batch_size=1 \
|
||||
# --max_height=480 \
|
||||
# --max_width=832 \
|
||||
# --num_frames=81 \
|
||||
# --dataloader_num_workers 1 \
|
||||
# --output_dir=$OUTPUT_DIR \
|
||||
# --train_fps 16
|
||||
|
||||
# torchrun --nproc_per_node=$GPU_NUM \
|
||||
# fastvideo/data_preprocess/preprocess_text_embeddings_v1.py \
|
||||
# --model_path $MODEL_PATH \
|
||||
# --output_dir=$OUTPUT_DIR
|
||||
|
||||
torchrun --nproc_per_node=1 \
|
||||
fastvideo/data_preprocess/preprocess_validation_text_embeddings_v1.py \
|
||||
--model_path $MODEL_PATH \
|
||||
--output_dir=$OUTPUT_DIR \
|
||||
--validation_prompt_txt $VALIDATION_PATH
|
||||
Reference in New Issue
Block a user