Davids048andClaude Opus 5.5
d051603677
- v1_preprocess.py and v1_preprocessing_new.py load a PreprocessRunConfig
from --config <yaml> plus dotted overrides; their argparse flags are
gone. The preprocessing pipelines, workflows, datasets, and stages read
resolved_config.preprocess.* and typed paths instead of
preprocess_config and an argparse namespace.
- The overfit preprocessors build resolved configs instead of FastVideoArgs.
- Each preprocessing launcher has a YAML file next to it with the values
its flags held; shell variables stay as dotted overrides. Against the
3f6893a0 snapshots of the 17 launchers, every value and PipelineConfig
matches, except mode (preprocess instead of inference), ltx2_vae_tiling,
and text_only's vae_config.load_encoder. Five launchers that the old
parser rejected now carry their intended values; the never-defined
--model_type and --validation_dataset_file flags are dropped.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012M91wnVFmPEvJ9h39r5BH7
79 lines
2.5 KiB
Bash
79 lines
2.5 KiB
Bash
#!/bin/bash
|
|
#SBATCH --partition=main
|
|
#SBATCH --nodes=1
|
|
#SBATCH --ntasks-per-node=8
|
|
#SBATCH --gres=gpu:8
|
|
#SBATCH --cpus-per-task=16
|
|
#SBATCH --mem=960G
|
|
#SBATCH --exclusive
|
|
#SBATCH --time=72:00:00
|
|
|
|
# conda init
|
|
# source ~/conda/miniconda/bin/activate
|
|
# PYTHON_VIRTUAL_ENVIRONMENT=fastvideo-train-yq
|
|
# conda activate $PYTHON_VIRTUAL_ENVIRONMENT
|
|
nvidia-smi
|
|
nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv
|
|
|
|
echo " "
|
|
echo " Number of nodes:= " $SLURM_JOB_NUM_NODES
|
|
echo " GPUs per node:= " $SLURM_JOB_GPUS
|
|
echo " Running on multiple nodes/GPU devices for TEXT-ONLY preprocessing"
|
|
echo ""
|
|
echo " Run started at:- "
|
|
date
|
|
|
|
# Accept parameters from launch script
|
|
START_FILE=${1:-1} # Starting file number for this node
|
|
NODE_ID=${2:-0} # Node identifier (0-7)
|
|
|
|
num_gpus=1
|
|
export MODEL_BASE=Wan-AI/Wan2.1-T2V-1.3B-Diffusers
|
|
# Start port number - we'll increment for each job
|
|
base_port=$((29603 + NODE_ID * 100)) # Different port range per node
|
|
|
|
# Create an array of CUDA device IDs
|
|
gpu_ids=(0 1 2 3 4 5 6 7)
|
|
|
|
GPU_NUM=1
|
|
MODEL_TYPE="wan"
|
|
|
|
echo "NODE_ID: $NODE_ID"
|
|
echo "START_FILE: $START_FILE"
|
|
echo "Base port for this node: $base_port"
|
|
echo "Processing TEXT-ONLY data"
|
|
|
|
# Run 8 parallel preprocessing jobs on this node
|
|
for i in {1..8}; do
|
|
# Calculate port for this job
|
|
port=$((base_port + i))
|
|
|
|
# Get GPU ID using modulo to cycle through available GPUs
|
|
gpu=${gpu_ids[((i-1))]}
|
|
|
|
# Calculate which file this GPU should process
|
|
file_num=$((START_FILE + i - 1))
|
|
DATA_MERGE_PATH="assets/prompts/v2m_${file_num}.txt"
|
|
|
|
# Create unique output directory based on node and GPU for text-only processing
|
|
OUTPUT_DIR="data/test-text-preprocessing/Node_${NODE_ID}_GPU_${i}_File_${file_num}"
|
|
|
|
start_cpu=$(( (i-1)*2 )) # Reduced CPU allocation for 8 nodes
|
|
end_cpu=$(( start_cpu+1 ))
|
|
|
|
echo "Starting GPU $gpu processing text-only file v2m_${file_num}.txt on port $port, output: $OUTPUT_DIR"
|
|
|
|
# Run the text-only preprocessing command in background
|
|
CUDA_VISIBLE_DEVICES=$gpu taskset -c ${start_cpu}-${end_cpu} torchrun --nnodes=1 --nproc_per_node=$GPU_NUM --master_port $port \
|
|
fastvideo/pipelines/preprocess/v1_preprocess.py \
|
|
--config scripts/preprocess/syn_text.yaml \
|
|
--model_path $MODEL_BASE \
|
|
--preprocess.data_merge_path $DATA_MERGE_PATH \
|
|
--preprocess.dataset_output_dir=$OUTPUT_DIR &
|
|
done
|
|
|
|
# Wait for all jobs on this node to complete
|
|
wait
|
|
|
|
echo "All text-only processing blocks completed!"
|