diff --git a/nodes/loader.py b/nodes/loader.py index d8da7ef..48be85d 100644 --- a/nodes/loader.py +++ b/nodes/loader.py @@ -241,18 +241,20 @@ class SDNQModelLoader: compiler_available = check_cpp_compiler_available() # Configure torch.compile error handling - # SDNQ quantized models use torch.compile for dequantization by default + # SDNQ quantized models use torch.compile for weight dequantization optimization if not compiler_available: - print("⚠ C++ compiler not detected - torch.compile will use fallback mode") - print(" Model will still run on GPU with quantized weights (same VRAM usage)") - print(" Compilation errors will be suppressed and execution will continue") + print("⚠ C++ compiler not detected - using fallback mode for SDNQ optimizations") + print(" ✓ Model attention: SDPA (GPU-accelerated, no compiler needed)") + print(" ✓ Weight dequantization: Eager mode (GPU, slightly slower)") + print(" ✓ VRAM usage: Same as with compiler") + print(" ⚠ Performance: ~10-20% slower than with compiler optimizations") - # Suppress compilation errors - let torch.compile fail gracefully and fall back to eager mode - # CRITICAL: We do NOT disable torch.compile entirely (that would force CPU execution) - # Instead, we let it TRY to compile, catch errors, and fall back to GPU eager mode + # Suppress compilation errors for SDNQ's dequantization code + # This allows graceful fallback to eager mode for weight dequantization + # while still using SDPA for model attention (set via attn_implementation="sdpa") torch._dynamo.config.suppress_errors = True - # Optional: Set verbose to False to reduce error spam + # Reduce error verbosity torch._dynamo.config.verbose = False # Pre-load cleanup to clear any leftover state from previous failed loads @@ -282,10 +284,15 @@ class SDNQModelLoader: print("Loading SDNQ model pipeline...", flush=True) print("This may take a moment for large models...", flush=True) + # Use SDPA (Scaled Dot Product Attention) for model attention layers + # SDPA is GPU-accelerated and widely supported (PyTorch 2.0+) + # Does NOT require C++ compiler (unlike torch.compile) + # Much faster than eager mode for attention operations pipeline = DiffusionPipeline.from_pretrained( model_path, torch_dtype=torch_dtype, local_files_only=is_local, + attn_implementation="sdpa", # Use SDPA for fast GPU attention (no compiler needed) ) print(f"✓ Pipeline loading complete", flush=True)