Compare commits

...
Author SHA1 Message Date
SolitaryThinker 4117feeea0 fix 2026-02-15 01:22:26 +00:00
William Lin 36bf37e9ba [bugfix] Fix failed kernel publish and SFT regressions (#1103) 2026-02-14 16:19:17 -08:00
10 changed files with 30 additions and 7 deletions
@@ -67,9 +67,9 @@ jobs:
- torch-version: '2.9.1'
cuda-version: '12.8.0'
torch-cuda-short: 'cu128'
- torch-version: '2.10.0'
cuda-version: '12.8.0'
torch-cuda-short: 'cu128'
# - torch-version: '2.10.0'
# cuda-version: '12.8.0'
# torch-cuda-short: 'cu128'
steps:
- name: Free up disk space
@@ -223,3 +223,6 @@ jobs:
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: fastvideo-kernel/dist/
# PyPI does not allow replacing an existing file with the same name.
# This makes re-runs idempotent by skipping files already uploaded.
skip-existing: true
@@ -19,6 +19,10 @@ if "A40" in device_name:
device_reference_folder = "A40" + device_reference_folder_suffix
elif "L40S" in device_name:
device_reference_folder = "L40S" + device_reference_folder_suffix
elif "H100" in device_name:
device_reference_folder = "H100" + device_reference_folder_suffix
elif "H200" in device_name:
device_reference_folder = "H200" + device_reference_folder_suffix
else:
# device_reference_folder = "L40S" + device_reference_folder_suffix
logger.warning(f"Unsupported device for ssim tests: {device_name}")
@@ -41,6 +41,10 @@ if "A40" in device_name:
device_reference_folder = "A40" + device_reference_folder_suffix
elif "L40S" in device_name:
device_reference_folder = "L40S" + device_reference_folder_suffix
elif "H100" in device_name:
device_reference_folder = "H100" + device_reference_folder_suffix
elif "H200" in device_name:
device_reference_folder = "H200" + device_reference_folder_suffix
else:
# device_reference_folder = "L40S" + device_reference_folder_suffix
logger.warning(f"Unsupported device for ssim tests: {device_name}")
@@ -47,6 +47,8 @@ elif "L40S" in device_name:
device_reference_folder = "L40S" + device_reference_folder_suffix
elif "H100" in device_name:
device_reference_folder = "H100" + device_reference_folder_suffix
elif "H200" in device_name:
device_reference_folder = "H200" + device_reference_folder_suffix
else:
device_reference_folder = None
logger.warning("Unsupported device for ssim tests: %s", device_name)
@@ -35,6 +35,8 @@ elif "L40S" in device_name:
device_reference_folder = "L40S" + device_reference_folder_suffix
elif "H100" in device_name:
device_reference_folder = "H100" + device_reference_folder_suffix
elif "H200" in device_name:
device_reference_folder = "H200" + device_reference_folder_suffix
else:
logger.warning(f"Unsupported device for ssim tests: {device_name}")
@@ -22,6 +22,10 @@ if "A40" in device_name:
device_reference_folder = "A40" + device_reference_folder_suffix
elif "L40S" in device_name:
device_reference_folder = "L40S" + device_reference_folder_suffix
elif "H100" in device_name:
device_reference_folder = "H100" + device_reference_folder_suffix
elif "H200" in device_name:
device_reference_folder = "H200" + device_reference_folder_suffix
else:
# device_reference_folder = "L40S" + device_reference_folder_suffix
logger.warning(f"Unsupported device for ssim tests: {device_name}")
@@ -22,6 +22,8 @@ def _device_reference_folder() -> str:
return "L40S" + suffix
if "H100" in device_name:
return "H100" + suffix
if "H200" in device_name:
return "H200" + suffix
if "RTX 4090" in device_name or "4090" in device_name:
return "RTX4090" + suffix
@@ -121,9 +121,9 @@ def test_distributed_training():
wandb_summary = json.load(open(summary_file))
fields_and_thresholds = {
'avg_step_time': 5,
'avg_step_time': 15,
'grad_norm': 0.5,
'step_time': 5,
'step_time': 15,
'train_loss': 0.04
}
@@ -136,9 +136,9 @@ def test_distributed_training():
wandb_summary = json.load(open(summary_file))
fields_and_thresholds = {
'avg_step_time': 6.0,
'avg_step_time': 15.0,
'grad_norm': 0.3,
'step_time': 6.0,
'step_time': 15.0,
'train_loss': 0.0025
}
+2
View File
@@ -19,6 +19,7 @@ from einops import rearrange
from torch.utils.data import DataLoader
from torchdata.stateful_dataloader import StatefulDataLoader
from tqdm.auto import tqdm
from diffusers import FlowMatchEulerDiscreteScheduler
import fastvideo.envs as envs
try:
@@ -605,6 +606,7 @@ class TrainingPipeline(LoRAPipeline, ABC):
device="cpu").manual_seed(self.seed + self.global_rank)
logger.info("Initialized random seeds with seed: %s",
self.seed + self.global_rank)
self.noise_scheduler = FlowMatchEulerDiscreteScheduler()
if self.training_args.resume_from_checkpoint:
self._resume_from_checkpoint()