🔧 INTEGRATE DEBUG SYSTEM: Add comprehensive data capture to ROCM nodes

- Integrated debug_config.py into nodes.py
- Added debug capture to ROCMOptimizedVAEDecode (input/output/timing/memory)
- Added debug capture to ROCMOptimizedKSampler (input/output/timing/memory)
- Debug data will be saved to test_data/captured/ when ROCM_NINODES_DEBUG=1
- Zero performance impact when debug mode disabled
- Ready to capture real workflow data for testing and optimization
This commit is contained in:
Gavroche
2025-10-11 10:36:24 -04:00
parent b3f0d7bd8d
commit 4d70c04bd6
+95
View File
@@ -14,6 +14,7 @@ import logging
import folder_paths
from typing import Dict, Any, Tuple, Optional
import time
from debug_config import DEBUG_MODE, save_debug_data, capture_timing, capture_memory_usage, log_debug
class ROCMOptimizedVAEDecode:
"""
@@ -85,6 +86,20 @@ class ROCMOptimizedVAEDecode:
Optimized VAE decode for ROCm/AMD GPUs with video support
"""
start_time = time.time()
log_debug(f"ROCMOptimizedVAEDecode.decode started with samples shape: {samples['samples'].shape}")
# Capture input data for debugging
if DEBUG_MODE:
save_debug_data(samples, "vae_decode_input", "flux_1024x1024", {
'node_type': 'ROCMOptimizedVAEDecode',
'tile_size': tile_size,
'overlap': overlap,
'use_rocm_optimizations': use_rocm_optimizations,
'precision_mode': precision_mode,
'batch_optimization': batch_optimization,
'video_chunk_size': video_chunk_size,
'memory_optimization_enabled': memory_optimization_enabled
})
# Get device information
device = vae.device
@@ -156,6 +171,27 @@ class ROCMOptimizedVAEDecode:
print(f"Converted to 4D format: {result.shape}")
print(f"Video decode completed: {result.shape}")
# Capture output data and timing for debugging
if DEBUG_MODE:
end_time = time.time()
save_debug_data(result, "vae_decode_output", "flux_1024x1024", {
'node_type': 'ROCMOptimizedVAEDecode',
'execution_time': end_time - start_time,
'output_shape': result.shape,
'output_dtype': str(result.dtype),
'output_device': str(result.device)
})
capture_timing("vae_decode", start_time, end_time, {
'node_type': 'ROCMOptimizedVAEDecode',
'is_video': True,
'video_chunks': len(chunk_results) if 'chunk_results' in locals() else 1
})
capture_memory_usage("vae_decode", {
'node_type': 'ROCMOptimizedVAEDecode',
'is_video': True
})
return (result,)
# Set optimal precision for AMD GPUs
@@ -317,6 +353,27 @@ class ROCMOptimizedVAEDecode:
if len(result.shape) == 4 and result.shape[1] == 3:
# Convert from (B, C, H, W) to (B, H, W, C) for video processing
result = result.permute(0, 2, 3, 1).contiguous()
# Capture output data and timing for debugging
if DEBUG_MODE:
end_time = time.time()
save_debug_data(result, "vae_decode_output", "flux_1024x1024", {
'node_type': 'ROCMOptimizedVAEDecode',
'execution_time': end_time - start_time,
'output_shape': result.shape,
'output_dtype': str(result.dtype),
'output_device': str(result.device)
})
capture_timing("vae_decode", start_time, end_time, {
'node_type': 'ROCMOptimizedVAEDecode',
'is_video': False,
'tile_size': tile_size,
'overlap': overlap
})
capture_memory_usage("vae_decode", {
'node_type': 'ROCMOptimizedVAEDecode',
'is_video': False
})
return result
@@ -575,6 +632,23 @@ class ROCMOptimizedKSampler:
Optimized sampling for ROCm/AMD GPUs
"""
start_time = time.time()
log_debug(f"ROCMOptimizedKSampler.sample started with latent shape: {latent_image['samples'].shape}")
# Capture input data for debugging
if DEBUG_MODE:
save_debug_data(latent_image, "ksampler_input", "flux_1024x1024", {
'node_type': 'ROCMOptimizedKSampler',
'seed': seed,
'steps': steps,
'cfg': cfg,
'sampler_name': sampler_name,
'scheduler': scheduler,
'denoise': denoise,
'use_rocm_optimizations': use_rocm_optimizations,
'precision_mode': precision_mode,
'memory_optimization': memory_optimization,
'attention_optimization': attention_optimization
})
# Get device information
device = model.model_dtype()
@@ -659,6 +733,27 @@ class ROCMOptimizedKSampler:
sample_time = time.time() - start_time
logging.info(f"ROCM KSampler completed in {sample_time:.2f}s")
# Capture output data and timing for debugging
if DEBUG_MODE:
save_debug_data(result, "ksampler_output", "flux_1024x1024", {
'node_type': 'ROCMOptimizedKSampler',
'execution_time': sample_time,
'output_shape': result[0]['samples'].shape if result and len(result) > 0 else None,
'output_dtype': str(result[0]['samples'].dtype) if result and len(result) > 0 else None,
'output_device': str(result[0]['samples'].device) if result and len(result) > 0 else None
})
capture_timing("ksampler", start_time, time.time(), {
'node_type': 'ROCMOptimizedKSampler',
'steps': steps,
'cfg': cfg,
'sampler_name': sampler_name,
'scheduler': scheduler
})
capture_memory_usage("ksampler", {
'node_type': 'ROCMOptimizedKSampler',
'steps': steps
})
return result