Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4117feeea0 | ||
|
|
36bf37e9ba |
@@ -67,9 +67,9 @@ jobs:
|
||||
- torch-version: '2.9.1'
|
||||
cuda-version: '12.8.0'
|
||||
torch-cuda-short: 'cu128'
|
||||
- torch-version: '2.10.0'
|
||||
cuda-version: '12.8.0'
|
||||
torch-cuda-short: 'cu128'
|
||||
# - torch-version: '2.10.0'
|
||||
# cuda-version: '12.8.0'
|
||||
# torch-cuda-short: 'cu128'
|
||||
|
||||
steps:
|
||||
- name: Free up disk space
|
||||
@@ -223,3 +223,6 @@ jobs:
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
packages-dir: fastvideo-kernel/dist/
|
||||
# PyPI does not allow replacing an existing file with the same name.
|
||||
# This makes re-runs idempotent by skipping files already uploaded.
|
||||
skip-existing: true
|
||||
|
||||
@@ -19,6 +19,10 @@ if "A40" in device_name:
|
||||
device_reference_folder = "A40" + device_reference_folder_suffix
|
||||
elif "L40S" in device_name:
|
||||
device_reference_folder = "L40S" + device_reference_folder_suffix
|
||||
elif "H100" in device_name:
|
||||
device_reference_folder = "H100" + device_reference_folder_suffix
|
||||
elif "H200" in device_name:
|
||||
device_reference_folder = "H200" + device_reference_folder_suffix
|
||||
else:
|
||||
# device_reference_folder = "L40S" + device_reference_folder_suffix
|
||||
logger.warning(f"Unsupported device for ssim tests: {device_name}")
|
||||
|
||||
@@ -41,6 +41,10 @@ if "A40" in device_name:
|
||||
device_reference_folder = "A40" + device_reference_folder_suffix
|
||||
elif "L40S" in device_name:
|
||||
device_reference_folder = "L40S" + device_reference_folder_suffix
|
||||
elif "H100" in device_name:
|
||||
device_reference_folder = "H100" + device_reference_folder_suffix
|
||||
elif "H200" in device_name:
|
||||
device_reference_folder = "H200" + device_reference_folder_suffix
|
||||
else:
|
||||
# device_reference_folder = "L40S" + device_reference_folder_suffix
|
||||
logger.warning(f"Unsupported device for ssim tests: {device_name}")
|
||||
|
||||
@@ -47,6 +47,8 @@ elif "L40S" in device_name:
|
||||
device_reference_folder = "L40S" + device_reference_folder_suffix
|
||||
elif "H100" in device_name:
|
||||
device_reference_folder = "H100" + device_reference_folder_suffix
|
||||
elif "H200" in device_name:
|
||||
device_reference_folder = "H200" + device_reference_folder_suffix
|
||||
else:
|
||||
device_reference_folder = None
|
||||
logger.warning("Unsupported device for ssim tests: %s", device_name)
|
||||
|
||||
@@ -35,6 +35,8 @@ elif "L40S" in device_name:
|
||||
device_reference_folder = "L40S" + device_reference_folder_suffix
|
||||
elif "H100" in device_name:
|
||||
device_reference_folder = "H100" + device_reference_folder_suffix
|
||||
elif "H200" in device_name:
|
||||
device_reference_folder = "H200" + device_reference_folder_suffix
|
||||
else:
|
||||
logger.warning(f"Unsupported device for ssim tests: {device_name}")
|
||||
|
||||
|
||||
@@ -22,6 +22,10 @@ if "A40" in device_name:
|
||||
device_reference_folder = "A40" + device_reference_folder_suffix
|
||||
elif "L40S" in device_name:
|
||||
device_reference_folder = "L40S" + device_reference_folder_suffix
|
||||
elif "H100" in device_name:
|
||||
device_reference_folder = "H100" + device_reference_folder_suffix
|
||||
elif "H200" in device_name:
|
||||
device_reference_folder = "H200" + device_reference_folder_suffix
|
||||
else:
|
||||
# device_reference_folder = "L40S" + device_reference_folder_suffix
|
||||
logger.warning(f"Unsupported device for ssim tests: {device_name}")
|
||||
|
||||
@@ -22,6 +22,8 @@ def _device_reference_folder() -> str:
|
||||
return "L40S" + suffix
|
||||
if "H100" in device_name:
|
||||
return "H100" + suffix
|
||||
if "H200" in device_name:
|
||||
return "H200" + suffix
|
||||
if "RTX 4090" in device_name or "4090" in device_name:
|
||||
return "RTX4090" + suffix
|
||||
|
||||
|
||||
@@ -121,9 +121,9 @@ def test_distributed_training():
|
||||
wandb_summary = json.load(open(summary_file))
|
||||
|
||||
fields_and_thresholds = {
|
||||
'avg_step_time': 5,
|
||||
'avg_step_time': 15,
|
||||
'grad_norm': 0.5,
|
||||
'step_time': 5,
|
||||
'step_time': 15,
|
||||
'train_loss': 0.04
|
||||
}
|
||||
|
||||
|
||||
@@ -136,9 +136,9 @@ def test_distributed_training():
|
||||
wandb_summary = json.load(open(summary_file))
|
||||
|
||||
fields_and_thresholds = {
|
||||
'avg_step_time': 6.0,
|
||||
'avg_step_time': 15.0,
|
||||
'grad_norm': 0.3,
|
||||
'step_time': 6.0,
|
||||
'step_time': 15.0,
|
||||
'train_loss': 0.0025
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ from einops import rearrange
|
||||
from torch.utils.data import DataLoader
|
||||
from torchdata.stateful_dataloader import StatefulDataLoader
|
||||
from tqdm.auto import tqdm
|
||||
from diffusers import FlowMatchEulerDiscreteScheduler
|
||||
|
||||
import fastvideo.envs as envs
|
||||
try:
|
||||
@@ -605,6 +606,7 @@ class TrainingPipeline(LoRAPipeline, ABC):
|
||||
device="cpu").manual_seed(self.seed + self.global_rank)
|
||||
logger.info("Initialized random seeds with seed: %s",
|
||||
self.seed + self.global_rank)
|
||||
self.noise_scheduler = FlowMatchEulerDiscreteScheduler()
|
||||
|
||||
if self.training_args.resume_from_checkpoint:
|
||||
self._resume_from_checkpoint()
|
||||
|
||||
Reference in New Issue
Block a user