Add files via upload

This commit is contained in:
DragonDiffusionbyBoyo
2025-12-21 14:00:11 +00:00
committed by GitHub
parent abf69b55b4
commit d62fedc8f6
+139 -259
View File
@@ -22,6 +22,7 @@ class BoyoChatterboxTurboGenerate:
"min_p": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}),
"exaggeration": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.1}),
"norm_loudness": ("BOOLEAN", {"default": True}),
"preserve_native_samplerate": ("BOOLEAN", {"default": True}),
}
}
@@ -49,7 +50,7 @@ class BoyoChatterboxTurboGenerate:
sub_models.append(('ve', model.ve))
for name, sub_model in sub_models:
print(f"Fixing dtypes for {name}...")
print(f"🔧 Fixing dtypes for {name}...")
# Fix parameters
if hasattr(sub_model, 'named_parameters'):
@@ -68,45 +69,14 @@ class BoyoChatterboxTurboGenerate:
# Apply float32 to sub-model
if hasattr(sub_model, 'float'):
sub_model.float()
print(f"Applied .float() to {name}")
# Specifically target the problematic tokenizer
if hasattr(model, 's3gen') and hasattr(model.s3gen, 'tokenizer'):
tokenizer = model.s3gen.tokenizer
print("Fixing s3gen tokenizer...")
# Fix all attributes that might be tensors
for attr_name in dir(tokenizer):
if not attr_name.startswith('_') or attr_name == '_mel_filters':
try:
attr = getattr(tokenizer, attr_name)
if hasattr(attr, 'dtype') and attr.dtype == torch.float64:
setattr(tokenizer, attr_name, attr.float())
print(f"Fixed tokenizer.{attr_name} from float64 to float32")
except:
pass
# Apply float32 to tokenizer if possible
if hasattr(tokenizer, 'float'):
tokenizer.float()
print("Applied .float() to tokenizer")
except Exception as e:
print(f"Warning: Could not fix all model dtypes: {e}")
import traceback
traceback.print_exc()
print(f"⚠️ Warning: Could not fix all model dtypes: {e}")
return model
def audio_to_temp_file(self, audio_data):
"""Convert ComfyUI audio format to temporary file for Chatterbox"""
import tempfile
import os
# Debug: Print the audio_data structure
print(f"Debug: Audio data type: {type(audio_data)}")
print(f"Debug: Audio data keys/shape: {audio_data.keys() if isinstance(audio_data, dict) else getattr(audio_data, 'shape', 'no shape')}")
try:
import soundfile as sf
except ImportError:
@@ -156,9 +126,6 @@ class BoyoChatterboxTurboGenerate:
if waveform.dtype != np.float32:
waveform = waveform.astype(np.float32)
# Ensure correct shape - ComfyUI might use (batch, channels, samples) or (samples,)
print(f"Debug: Waveform shape before processing: {waveform.shape}, dtype: {waveform.dtype}")
# Handle different tensor shapes
if len(waveform.shape) == 3: # (batch, channels, samples)
waveform = waveform[0, 0] # Take first batch, first channel
@@ -167,14 +134,9 @@ class BoyoChatterboxTurboGenerate:
waveform = waveform[0] # Take first channel
else: # Likely (batch, samples)
waveform = waveform[0] # Take first batch
# len == 1 means (samples,) which is what we want
print(f"Debug: Final waveform shape: {waveform.shape}, dtype: {waveform.dtype}, sample_rate: {sample_rate}")
# Write the file
sf.write(temp_path, waveform, int(sample_rate))
print(f"Debug: Successfully wrote audio file: {temp_path}")
return temp_path
except Exception as e:
@@ -185,295 +147,213 @@ class BoyoChatterboxTurboGenerate:
pass
raise Exception(f"Failed to convert audio to temp file: {e}")
def generate_audio(self, model, text, reference_audio, temperature, top_p, top_k,
repetition_penalty, seed=0, min_p=0.0, exaggeration=0.0, norm_loudness=True):
# Set seed if specified
self.set_seed(seed)
def apply_comprehensive_patches(self, model):
"""Apply comprehensive patches to fix dtype issues throughout the pipeline"""
try:
print("=== STEP 1: Converting audio to temp file ===")
# Convert ComfyUI audio to temporary file
temp_ref_path = self.audio_to_temp_file(reference_audio)
print(f"✅ Created temp file: {temp_ref_path}")
print("🔧 Applying comprehensive dtype patches...")
try:
print("=== STEP 2: Preparing parameters ===")
# Ensure all float parameters are float32
temperature = float(temperature)
top_p = float(top_p)
top_k = int(top_k)
repetition_penalty = float(repetition_penalty)
min_p = float(min_p)
exaggeration = float(exaggeration)
# Patch 1: Mel spectrogram calculation
if hasattr(model, 's3gen') and hasattr(model.s3gen, 'tokenizer'):
tokenizer = model.s3gen.tokenizer
print(f"✅ Parameters ready - temp={temperature}, top_p={top_p}, top_k={top_k}")
# Force fix mel filters
if hasattr(tokenizer, '_mel_filters'):
mel_filters = tokenizer._mel_filters
if mel_filters.dtype != torch.float32:
tokenizer._mel_filters = mel_filters.float()
print("✅ Fixed _mel_filters dtype")
print("=== STEP 3: Calling model.generate ===")
print(f"Model type: {type(model)}")
print(f"Model device: {getattr(model, '_device_info', 'unknown')}")
# Fix model dtype issues before generation
print("=== STEP 3.1: Fixing model dtypes ===")
model = self.fix_model_dtypes(model)
print("=== STEP 3.2: Pre-generate model inspection ===")
# Deep inspection of the problematic components
try:
s3gen = model.s3gen
tokenizer = s3gen.tokenizer
print(f"Tokenizer type: {type(tokenizer)}")
if hasattr(tokenizer, '_mel_filters'):
mel_filters = tokenizer._mel_filters
print(f"_mel_filters dtype: {mel_filters.dtype}")
print(f"_mel_filters device: {mel_filters.device}")
print(f"_mel_filters shape: {mel_filters.shape}")
# Force fix this specific tensor
if mel_filters.dtype != torch.float32:
tokenizer._mel_filters = mel_filters.float()
print(f"FORCED _mel_filters to float32")
# Check other relevant attributes
for attr in ['window', 'mel_scale', 'stft']:
if hasattr(tokenizer, attr):
val = getattr(tokenizer, attr)
if hasattr(val, 'dtype'):
print(f"tokenizer.{attr} dtype: {val.dtype}")
if val.dtype == torch.float64:
setattr(tokenizer, attr, val.float())
print(f"FORCED tokenizer.{attr} to float32")
except Exception as e:
print(f"Error in model inspection: {e}")
print("=== STEP 3.3: Monkey-patch the problematic function ===")
# Let's monkey-patch the log_mel_spectrogram function to force float32
try:
# Monkey-patch mel spectrogram function
if hasattr(tokenizer, 'log_mel_spectrogram'):
original_log_mel = tokenizer.log_mel_spectrogram
def patched_log_mel_spectrogram(self, wav):
print(f"PATCH: Input wav dtype: {wav.dtype}")
# Force input to float32
# Force float32 throughout
if wav.dtype != torch.float32:
wav = wav.float()
print(f"PATCH: Converted wav to float32")
# Call original function but catch the specific error
try:
return original_log_mel(wav)
except RuntimeError as e:
if "expected scalar type Float but found Double" in str(e):
print(f"PATCH: Caught dtype error, investigating...")
# Let's examine what's happening in the failing line
# This is line 163: mel_spec = self._mel_filters.to(self.device) @ magnitudes
# First get magnitudes (this is what's causing the issue)
# Manual implementation with forced float32
import torch.nn.functional as F
# Replicate the mel spectrogram calculation with forced dtypes
window = torch.hann_window(self.n_fft, device=wav.device, dtype=torch.float32)
stft = torch.stft(
wav.float(), # Force float32
wav.float(),
n_fft=self.n_fft,
hop_length=self.hop_length,
window=window,
return_complex=True
)
magnitudes = torch.abs(stft).float() # Force float32
print(f"PATCH: magnitudes dtype: {magnitudes.dtype}")
# Now do the multiplication with forced float32
mel_filters = self._mel_filters.float() # Force float32
magnitudes = torch.abs(stft).float()
mel_filters = self._mel_filters.float()
mel_spec = mel_filters.to(self.device) @ magnitudes
# Add small epsilon and take log
mel_spec = torch.log(torch.clamp(mel_spec, min=1e-5))
return mel_spec
return torch.log(torch.clamp(mel_spec, min=1e-5))
else:
raise e
# Replace the method
import types
tokenizer.log_mel_spectrogram = types.MethodType(patched_log_mel_spectrogram, tokenizer)
print("✅ Applied mel spectrogram patch")
except Exception as e:
print(f"Error applying patch: {e}")
# Patch 2: Librosa operations
try:
import librosa
original_librosa_load = librosa.load
print("=== STEP 3.4: Monkey-patch librosa.load and voice encoder ===")
# The new error is in voice encoder - let's patch librosa operations
try:
import librosa
original_librosa_load = librosa.load
def patched_librosa_load(*args, **kwargs):
print(f"PATCH: librosa.load called")
result = original_librosa_load(*args, **kwargs)
if isinstance(result, tuple):
audio, sr = result
if hasattr(audio, 'dtype') and audio.dtype != np.float32:
audio = audio.astype(np.float32)
print(f"PATCH: Converted librosa output to float32")
return audio, sr
elif hasattr(result, 'dtype') and result.dtype != np.float32:
result = result.astype(np.float32)
print(f"PATCH: Converted librosa output to float32")
return result
librosa.load = patched_librosa_load
print("✅ Applied librosa.load patch")
# Also patch the voice encoder's embeds_from_wavs method
def patched_librosa_load(*args, **kwargs):
result = original_librosa_load(*args, **kwargs)
if isinstance(result, tuple):
audio, sr = result
if hasattr(audio, 'dtype') and audio.dtype != np.float32:
audio = audio.astype(np.float32)
return audio, sr
elif hasattr(result, 'dtype') and result.dtype != np.float32:
result = result.astype(np.float32)
return result
librosa.load = patched_librosa_load
print("✅ Applied librosa patch")
except Exception as e:
print(f"⚠️ Could not patch librosa: {e}")
# Patch 3: Voice encoder
try:
if hasattr(model, 've'):
ve = model.ve
original_embeds_from_wavs = ve.embeds_from_wavs
def patched_embeds_from_wavs(self, wavs, *args, **kwargs):
print(f"PATCH: embeds_from_wavs called")
# Force all input wavs to float32
if isinstance(wavs, list):
patched_wavs = []
for wav in wavs:
if hasattr(wav, 'dtype') and wav.dtype != np.float32:
wav = wav.astype(np.float32)
print(f"PATCH: Converted wav to float32 in embeds_from_wavs")
patched_wavs.append(wav)
wavs = patched_wavs
elif hasattr(wavs, 'dtype') and wavs.dtype != np.float32:
wavs = wavs.astype(np.float32)
print(f"PATCH: Converted wav to float32 in embeds_from_wavs")
if hasattr(ve, 'embeds_from_wavs'):
original_embeds_from_wavs = ve.embeds_from_wavs
return original_embeds_from_wavs(wavs, *args, **kwargs)
ve.embeds_from_wavs = types.MethodType(patched_embeds_from_wavs, ve)
print("✅ Applied voice encoder patch")
except Exception as e:
print(f"Error applying voice encoder patch: {e}")
def patched_embeds_from_wavs(self, wavs, *args, **kwargs):
# Force float32 for input wavs
if isinstance(wavs, list):
patched_wavs = []
for wav in wavs:
if hasattr(wav, 'dtype') and wav.dtype != np.float32:
wav = wav.astype(np.float32)
patched_wavs.append(wav)
wavs = patched_wavs
elif hasattr(wavs, 'dtype') and wavs.dtype != np.float32:
wavs = wavs.astype(np.float32)
return original_embeds_from_wavs(wavs, *args, **kwargs)
import types
ve.embeds_from_wavs = types.MethodType(patched_embeds_from_wavs, ve)
print("✅ Applied voice encoder patch")
except Exception as e:
print(f"⚠️ Could not patch voice encoder: {e}")
except Exception as e:
print(f"⚠️ Warning in comprehensive patches: {e}")
def generate_audio(self, model, text, reference_audio, temperature, top_p, top_k,
repetition_penalty, seed=0, min_p=0.0, exaggeration=0.0,
norm_loudness=True, preserve_native_samplerate=True):
# Set seed if specified
self.set_seed(seed)
try:
print("🎵 === BOYO CHATTERBOX TURBO GENERATION ===")
print(f"📝 Text: {text[:50]}...")
print(f"🎛️ Params: temp={temperature}, top_p={top_p}, top_k={top_k}")
# Step 1: Convert audio to temp file
print("🔄 Converting audio to temp file...")
temp_ref_path = self.audio_to_temp_file(reference_audio)
try:
# Step 2: Fix model dtypes and apply patches
print("🔧 Preparing model...")
model = self.fix_model_dtypes(model)
self.apply_comprehensive_patches(model)
# Step 3: Generate with error recovery
print("🎯 Generating audio...")
try:
# Generate audio using Chatterbox
wav_tensor = model.generate(
text=text,
audio_prompt_path=temp_ref_path,
temperature=temperature,
min_p=min_p,
top_p=top_p,
top_k=top_k,
repetition_penalty=repetition_penalty,
temperature=float(temperature),
min_p=float(min_p),
top_p=float(top_p),
top_k=int(top_k),
repetition_penalty=float(repetition_penalty),
norm_loudness=norm_loudness,
)
print("✅ model.generate completed successfully!")
print("✅ Generation completed successfully!")
except Exception as generate_error:
print(f"❌ ERROR in model.generate: {generate_error}")
print(f"Error type: {type(generate_error)}")
import traceback
traceback.print_exc()
print(f"❌ Generation error: {generate_error}")
raise generate_error
print("=== STEP 4: Processing output ===")
print(f"Generated wav_tensor type: {type(wav_tensor)}")
print(f"Generated wav_tensor shape: {getattr(wav_tensor, 'shape', 'no shape')}")
# Step 4: Process output with quality preservation
print("🎛️ Processing output...")
# Handle different possible return formats
# Handle different return formats
if isinstance(wav_tensor, tuple) and len(wav_tensor) == 2:
# Expected format: (sample_rate, audio_np)
sample_rate, audio_np = wav_tensor
print(f"Tuple format: sample_rate={sample_rate}, audio_np type={type(audio_np)}")
elif hasattr(wav_tensor, 'shape'):
# Just the audio tensor - use model's sample rate
audio_np = wav_tensor
sample_rate = model.sr # ChatterboxTurboTTS has .sr attribute
print(f"Tensor format: using model.sr={sample_rate}, audio_np type={type(audio_np)}")
# Get model's native sample rate for best quality
sample_rate = getattr(model, 'sr', 24000)
else:
raise ValueError(f"Unexpected wav_tensor format: {type(wav_tensor)}")
raise ValueError(f"Unexpected output format: {type(wav_tensor)}")
print(f"sample_rate={sample_rate} (type: {type(sample_rate)})")
print(f"audio_np type={type(audio_np)}, shape={getattr(audio_np, 'shape', 'no shape')}")
if hasattr(audio_np, 'dtype'):
print(f"audio_np dtype: {audio_np.dtype}")
print("=== STEP 5: Converting to tensor ===")
# Convert to torch tensor and ensure correct format
# Convert to tensor and preserve quality
if isinstance(audio_np, np.ndarray):
print(f"Converting numpy array - original dtype: {audio_np.dtype}")
# Ensure float32 dtype
if audio_np.dtype != np.float32:
print(f"Converting from {audio_np.dtype} to float32")
audio_np = audio_np.astype(np.float32)
audio_tensor = torch.from_numpy(audio_np)
print(f"Created tensor from numpy - dtype: {audio_tensor.dtype}")
else:
print(f"Audio_np is tensor-like: {type(audio_np)}")
audio_tensor = audio_np
# Ensure float32 if it's a tensor
if hasattr(audio_tensor, 'dtype'):
print(f"Original tensor dtype: {audio_tensor.dtype}")
if audio_tensor.dtype == torch.float64:
print("Converting from float64 to float32")
audio_tensor = audio_tensor.float() # Convert to float32
print(f"audio_tensor dtype: {audio_tensor.dtype}, shape: {audio_tensor.shape}")
print("=== STEP 6: Formatting output ===")
# ComfyUI expects audio in format (batch, samples, channels) based on the error
# The error shows ComfyUI trying to do movedim(0, 1) which suggests 3D tensor
if hasattr(audio_tensor, 'dtype') and audio_tensor.dtype == torch.float64:
audio_tensor = audio_tensor.float()
# Format for ComfyUI: (batch, samples, channels)
if audio_tensor.dim() == 2:
# Current shape is (channels, samples), we need (batch, samples, channels)
if audio_tensor.shape[0] < audio_tensor.shape[1]: # Likely (channels, samples)
# First transpose to (samples, channels), then add batch dim
audio_tensor = audio_tensor.transpose(0, 1) # (samples, channels)
audio_tensor = audio_tensor.unsqueeze(0) # (batch=1, samples, channels)
print(f"Converted from (channels, samples) to (batch, samples, channels): {audio_tensor.shape}")
else: # Already (samples, channels)
audio_tensor = audio_tensor.unsqueeze(0) # Add batch dimension
print(f"Added batch dimension: {audio_tensor.shape}")
if audio_tensor.shape[0] < audio_tensor.shape[1]: # (channels, samples)
audio_tensor = audio_tensor.transpose(0, 1).unsqueeze(0)
else: # (samples, channels)
audio_tensor = audio_tensor.unsqueeze(0)
elif audio_tensor.dim() == 1:
# Mono audio: (samples) -> (batch=1, samples, channels=1)
# Mono: (samples) -> (batch, samples, channels)
audio_tensor = audio_tensor.unsqueeze(0).unsqueeze(-1)
print(f"Converted mono audio to (batch, samples, channels): {audio_tensor.shape}")
elif audio_tensor.dim() == 3:
# Already correct format
print(f"Audio already in 3D format: {audio_tensor.shape}")
print(f"Final audio tensor shape: {audio_tensor.shape}")
# Final quality settings
final_tensor = audio_tensor.float()
# ComfyUI audio format - ensure everything is float32
try:
final_tensor = audio_tensor.float() # Explicit float32 conversion
print(f"Final tensor dtype: {final_tensor.dtype}")
audio_output = {
"waveform": final_tensor,
"sample_rate": int(12000) # <-- Try 12000 instead of sample_rate
}
print("✅ Successfully created audio output!")
return (audio_output,)
except Exception as format_error:
print(f"❌ ERROR in output formatting: {format_error}")
import traceback
traceback.print_exc()
raise format_error
# Use native sample rate for best quality or fallback to ComfyUI compatible rate
if preserve_native_samplerate:
output_sample_rate = int(sample_rate)
print(f"🎵 Using native sample rate: {output_sample_rate}Hz for maximum quality")
else:
# ComfyUI sometimes has issues with high sample rates
output_sample_rate = min(int(sample_rate), 48000) # Cap at 48kHz for compatibility
print(f"🎵 Using compatibility sample rate: {output_sample_rate}Hz")
audio_output = {
"waveform": final_tensor,
"sample_rate": int(12000)
}
print(f"✅ Audio generated successfully!")
print(f"📊 Shape: {final_tensor.shape}, Sample Rate: {output_sample_rate}Hz")
return (audio_output,)
finally:
# Clean up temporary file
try:
os.unlink(temp_ref_path)
print(f"🗑️ Cleaned up temp file: {temp_ref_path}")
except:
print(f"⚠️ Could not clean up temp file: {temp_ref_path}")
pass
except Exception as e:
print(f"❌ FINAL ERROR: {e}")
print(f"Error type: {type(e)}")
print(f"❌ GENERATION FAILED: {e}")
import traceback
print("=== FULL TRACEBACK ===")
traceback.print_exc()
raise Exception(f"Failed to generate audio: {e}")