Davids048andClaude Opus 5.5
d051603677
- v1_preprocess.py and v1_preprocessing_new.py load a PreprocessRunConfig
from --config <yaml> plus dotted overrides; their argparse flags are
gone. The preprocessing pipelines, workflows, datasets, and stages read
resolved_config.preprocess.* and typed paths instead of
preprocess_config and an argparse namespace.
- The overfit preprocessors build resolved configs instead of FastVideoArgs.
- Each preprocessing launcher has a YAML file next to it with the values
its flags held; shell variables stay as dotted overrides. Against the
3f6893a0 snapshots of the 17 launchers, every value and PipelineConfig
matches, except mode (preprocess instead of inference), ltx2_vae_tiling,
and text_only's vae_config.load_encoder. Five launchers that the old
parser rejected now carry their intended values; the never-defined
--model_type and --validation_dataset_file flags are dropped.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012M91wnVFmPEvJ9h39r5BH7
47 lines
2.1 KiB
Python
47 lines
2.1 KiB
Python
from typing import TYPE_CHECKING, Optional
|
|
|
|
from tqdm import tqdm
|
|
|
|
from fastvideo.pipelines.pipeline_batch_info import PreprocessBatch
|
|
from fastvideo.workflow.preprocess.components import ParquetDatasetSaver
|
|
from fastvideo.workflow.preprocess.preprocess_workflow import PreprocessWorkflow
|
|
|
|
if TYPE_CHECKING:
|
|
from torch.utils.data import DataLoader
|
|
|
|
from fastvideo.pipelines.composed_pipeline_base import ComposedPipelineBase
|
|
from fastvideo.workflow.preprocess.components import (VideoForwardBatchBuilder)
|
|
|
|
|
|
class PreprocessWorkflowT2V(PreprocessWorkflow):
|
|
training_dataloader: "DataLoader"
|
|
validation_dataloader: Optional["DataLoader"]
|
|
preprocess_pipeline: "ComposedPipelineBase"
|
|
processed_dataset_saver: "ParquetDatasetSaver"
|
|
video_forward_batch_builder: "VideoForwardBatchBuilder"
|
|
|
|
def run(self) -> None:
|
|
# Training dataset preprocessing
|
|
for batch in tqdm(self.training_dataloader, desc="Preprocessing training dataset", unit="batch"):
|
|
forward_batch: PreprocessBatch = self.video_forward_batch_builder(batch)
|
|
|
|
forward_batch = self.preprocess_pipeline.forward(forward_batch, self.preprocess_pipeline.resolved_config)
|
|
|
|
self.processed_dataset_saver.save_and_write_parquet_batch(forward_batch, self.training_dataset_output_dir)
|
|
|
|
self.processed_dataset_saver.flush_tables()
|
|
self.processed_dataset_saver.clean_up()
|
|
|
|
# Validation dataset preprocessing
|
|
if self.validation_dataloader is not None:
|
|
for batch in tqdm(self.validation_dataloader, desc="Preprocessing validation dataset", unit="batch"):
|
|
forward_batch = self.video_forward_batch_builder(batch)
|
|
|
|
forward_batch = self.preprocess_pipeline.forward(forward_batch,
|
|
self.preprocess_pipeline.resolved_config)
|
|
|
|
self.processed_dataset_saver.save_and_write_parquet_batch(forward_batch,
|
|
self.validation_dataset_output_dir)
|
|
self.processed_dataset_saver.flush_tables()
|
|
self.processed_dataset_saver.clean_up()
|