diff --git a/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_k.py b/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_k.py index 5fc97f5ee..e8360957c 100644 --- a/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_k.py +++ b/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_k.py @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. """ import torch -import fp4quant +import fp4quant_cuda as fp4quant from triton.tools.mxfp import MXFP4Tensor from bench_utils import bench_kineto diff --git a/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_q.py b/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_q.py index 1d0576d59..f05c292e3 100644 --- a/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_q.py +++ b/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_q.py @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. """ import torch -import fp4quant +import fp4quant_cuda as fp4quant from triton.tools.mxfp import MXFP4Tensor from bench_utils import bench_kineto diff --git a/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_v.py b/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_v.py index 0592aa888..a17885ec2 100644 --- a/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_v.py +++ b/fastvideo-kernel/attn_qat_infer/quantization/bench/bench_quant_v.py @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. """ import torch -import fp4quant +import fp4quant_cuda as fp4quant from triton.tools.mxfp import MXFP4Tensor from bench_utils import bench_kineto diff --git a/fastvideo-kernel/csrc/turbodiffusion/quant/quant.cu b/fastvideo-kernel/csrc/turbodiffusion/quant/quant.cu index 9a56e3881..672013047 100644 --- a/fastvideo-kernel/csrc/turbodiffusion/quant/quant.cu +++ b/fastvideo-kernel/csrc/turbodiffusion/quant/quant.cu @@ -31,6 +31,11 @@ auto quant( std::optional& Output_S ) { + TORCH_CHECK( + Input.scalar_type() == torch::kHalf || Input.scalar_type() == torch::kBFloat16, + "Unsupported input data type for quant_cuda (int8 quantization)." + ); + using ElementOut = int8_t; static constexpr int BlockSize = 128; static constexpr int NumThrPerCta = 256; @@ -62,8 +67,7 @@ auto quant( } default: { - std::cerr << "Observing: " << Input.scalar_type() << " for the input datatype which is invalid"; - throw std::runtime_error("Unsupported input data type for quantize_to_fp4."); + TORCH_CHECK(false, "Unsupported input data type for quant_cuda (int8 quantization)."); } } diff --git a/fastvideo-kernel/python/fastvideo_kernel/triton_kernels/attn_qat_train.py b/fastvideo-kernel/python/fastvideo_kernel/triton_kernels/attn_qat_train.py index 52ef4294c..0fd0aedb9 100644 --- a/fastvideo-kernel/python/fastvideo_kernel/triton_kernels/attn_qat_train.py +++ b/fastvideo-kernel/python/fastvideo_kernel/triton_kernels/attn_qat_train.py @@ -1054,6 +1054,15 @@ class _attention(torch.autograd.Function): use_global_sf_P=True, use_global_sf_QKV=True, ): + if smooth_q: + # Not implemented, not merely disabled: only the backward kernels + # carry a SMOOTH_Q path (_attn_fwd has none), so restoring the + # triton_group_mean wiring would yield silently wrong gradients + # rather than a feature. The flag is kept for signature + # compatibility only. + raise NotImplementedError("smooth_q=True is not supported: the forward kernel has no " + "SMOOTH_Q path, so enabling it would produce incorrect gradients") + # shape constraints HEAD_DIM_Q, HEAD_DIM_K = q.shape[-1], k.shape[-1] # when v is in float8_e5m2 it is transposed. @@ -1364,10 +1373,8 @@ class _attention(torch.autograd.Function): BLOCK_M=PRE_BLOCK, HEAD_DIM=ctx.HEAD_DIM) + # smooth_q is rejected in forward(); ctx.smooth_q is always False here. q_m = None - if ctx.smooth_q: - # _, q_m = triton_group_mean(q) - q_m = q_m.repeat_interleave(q.shape[2] // q_m.shape[2], dim=2) # B,H,L,D sm100_optimized_backward = (getattr(ctx, "sm100_optimized", False) and ctx.use_qat_qkv_backward and not ctx.smooth_k and not ctx.smooth_q and N_CTX_KV % 16 == 0) diff --git a/fastvideo-kernel/tests/test_attn_qat_train.py b/fastvideo-kernel/tests/test_attn_qat_train.py index 22755e832..e221b50aa 100644 --- a/fastvideo-kernel/tests/test_attn_qat_train.py +++ b/fastvideo-kernel/tests/test_attn_qat_train.py @@ -4,6 +4,7 @@ Simple test for attn_qat_train. Tests forward and backward passes with and without QAT enabled. """ +import pytest import torch from fastvideo_kernel.triton_kernels.attn_qat_train import _attention from fastvideo_kernel.triton_kernels.fused_attention import attention as fused_attention @@ -13,6 +14,27 @@ attention = _attention.apply DEVICE = torch.device("cuda") +def test_smooth_q_fails_before_tensor_or_kernel_work(): + dummy = torch.empty((1, 1, 1, 16)) + + with pytest.raises(NotImplementedError, match="smooth_q=True is not supported"): + attention( + dummy, + dummy, + dummy, + False, + 1.0, + True, + True, + True, + True, + True, + True, + False, + True, + ) + + def attn_qat_train_wrapper(q_BLHD, k_BLHD, v_BLHD, is_causal=False, sm_scale=None): """ Wrapper function that mimics attn_qat_train from the backend wrapper. diff --git a/fastvideo-kernel/tests/test_nvfp4_benchmark_imports.py b/fastvideo-kernel/tests/test_nvfp4_benchmark_imports.py new file mode 100644 index 000000000..44f932ade --- /dev/null +++ b/fastvideo-kernel/tests/test_nvfp4_benchmark_imports.py @@ -0,0 +1,21 @@ +import ast +from pathlib import Path + +import pytest + + +BENCH_DIR = Path(__file__).resolve().parents[1] / "attn_qat_infer" / "quantization" / "bench" + + +@pytest.mark.parametrize("script_name", ["bench_quant_q.py", "bench_quant_k.py", "bench_quant_v.py"]) +def test_quant_benchmark_imports_built_extension(script_name: str) -> None: + tree = ast.parse((BENCH_DIR / script_name).read_text()) + imports = { + (alias.name, alias.asname) + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } + + assert ("fp4quant_cuda", "fp4quant") in imports + assert ("fp4quant", None) not in imports diff --git a/fastvideo-kernel/tests/test_turbodiffusion.py b/fastvideo-kernel/tests/test_turbodiffusion.py index 771ab7ffa..02ae43bdf 100644 --- a/fastvideo-kernel/tests/test_turbodiffusion.py +++ b/fastvideo-kernel/tests/test_turbodiffusion.py @@ -26,6 +26,14 @@ def rms_norm_ref(x, w, eps=1e-6): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") class TestTurboDiffusion: + def test_quant_invalid_dtype_names_int8_operation(self): + if turbodiffusion_ops.quant_cuda is None: + pytest.skip("quant_cuda not available") + + x = torch.ones((16, 128), dtype=torch.float32, device="cuda") + with pytest.raises(RuntimeError, match=r"Unsupported input data type for quant_cuda \(int8 quantization\)\."): + turbodiffusion_ops.quant_cuda(x, None, None) + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) @pytest.mark.parametrize("shape", [(16, 128), (32, 256), (1, 1024)]) def test_quant_correctness(self, dtype, shape):