added hubert safetensors

custom_nodes/loaders.py

@@ -88,10 +88,10 @@ class LoadPitchExtractionParams:
 class LoadHubertModel:
     @classmethod
     def INPUT_TYPES(cls):
-        model_list = ["hubert_base.pt"] + get_filenames(root=BASE_MODELS_DIR,folder="*",exts=["pt"],format_func=os.path.basename)
+        model_list = ["content-vec-best.safetensors"] + get_filenames(root=BASE_MODELS_DIR,folder="*",exts=["pt","safetensors"],format_func=os.path.basename)
         return {
             'required': {
-                'model': (model_list,{"default": "hubert_base.pt"}),
+                'model': (model_list,{"default": "content-vec-best.safetensors"}),
             },
         }
 
@@ -253,7 +253,7 @@ class LoadAudio:
     FUNCTION = "load_audio"
 
     def load_audio(self, audio, sr):
-        audio_path = folder_paths.get_annotated_filepath(audio)
+        audio_path = os.path.join(input_path,audio) #folder_paths.get_annotated_filepath(audio)
         widgetId = get_hash(audio_path)
         audio_name = os.path.basename(audio).split(".")[0]
         sr = None if sr=="None" else int(sr)
@@ -262,15 +262,13 @@ class LoadAudio:
 
     @classmethod
     def IS_CHANGED(cls, audio):
-        audio_path = folder_paths.get_annotated_filepath(audio)
+        audio_path = os.path.join(input_path,audio)
         print(f"{audio_path=}")
         return get_file_hash(audio_path)
     
 class DownloadAudio:
     @classmethod
     def INPUT_TYPES(cls):
-        input_dir = input_path
-        files = get_filenames(root=input_dir,exts=SUPPORTED_AUDIO,format_func=os.path.basename)
         
         return {
             "required": {

lib/audio.py

@@ -1,9 +1,8 @@
 import base64
 import io
 import os
-from typing import Union
 import zlib
-from .utils import get_hash, get_merge_func
+from .utils import get_merge_func
 import numpy as np
 import librosa
 import soundfile as sf
@@ -40,8 +39,8 @@ def load_audio(file, sr, **kwargs):
             .output("-", format="f32le", acodec="pcm_f32le", ac=1, ar=sr)
             .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True)
         )
-    except Exception as e:
-        raise RuntimeError(f"Failed to load audio: {e}")
+    except Exception as error:
+        raise RuntimeError(f"Failed to load audio: {error=} {file=}")
 
     return remix_audio((np.frombuffer(out, np.float32).flatten(), sr),**kwargs)
 

lib/infer_pack/loaders.py

@@ -0,0 +1,41 @@
+import torch
+from transformers import HubertModel, HubertConfig
+import safetensors.torch as st
+from safetensors import safe_open
+import json
+
+class HubertModelWithFinalProj(HubertModel):
+    def __init__(self, config):
+        super().__init__(config)
+
+        # The final projection layer is only used for backward compatibility.
+        # Following https://github.com/auspicious3000/contentvec/issues/6
+        # Remove this layer is necessary to achieve the desired outcome.
+        self.final_proj = torch.nn.Linear(config.hidden_size, config.classifier_proj_size)
+
+    @staticmethod
+    def load_safetensors(path: str, device="cpu"):
+        assert path.endswith(".safetensors"), f"{path} must end with '.safetensors'"
+        
+        with safe_open(path, framework="pt", device="cpu") as f:
+            metadata = f.metadata()
+            state_dict = {}
+            for key in f.keys():
+                state_dict[key] = f.get_tensor(key)
+        model = HubertModelWithFinalProj(HubertConfig.from_dict(json.loads(metadata["config"])))
+        model.load_state_dict(state_dict=state_dict)
+        return model.to(device)
+    
+    def save_safetensors(self, path: str):
+        assert path.endswith(".safetensors"), f"{path} must end with '.safetensors'"
+        
+        with open(path,"wb") as f:
+            state_dict = self.state_dict()
+            f.write(st.save(state_dict,dict(config=json.dumps(self.config.to_dict()))))
+
+    def extract_features(self, source: torch.Tensor, version="v2", **kwargs):
+        with torch.no_grad():
+            output_layer = 9 if version == "v1" else 12
+            output = self(source.to(self.config.torch_dtype), output_hidden_states=True)["hidden_states"][output_layer]
+            features = self.final_proj(output) if version == "v1" else output
+        return features.to(source.dtype)

lib/model_utils.py

@@ -1,10 +1,11 @@
 
 import hashlib
-import os
 import torch.nn.functional as F
 import librosa
 import torch
 
+from .infer_pack.loaders import HubertModelWithFinalProj
+
 def get_hash(model_path):
     try:
         with open(model_path, 'rb') as f:
@@ -15,18 +16,21 @@ def get_hash(model_path):
 
     return model_hash
 
-def load_hubert(model_path,config):
+def load_hubert(model_path: str, config):
     try:
-        from fairseq import checkpoint_utils
-        models, _, _ = checkpoint_utils.load_model_ensemble_and_task([model_path],suffix="",)
-        hubert_model = models[0]
-        hubert_model = hubert_model.to(config.device)
-        if config.is_half:
-            hubert_model = hubert_model.half()
+        if model_path.endswith(".safetensors"):
+            return HubertModelWithFinalProj.load_safetensors(model_path, device=config.device)
         else:
-            hubert_model = hubert_model.float()
-        hubert_model.eval()
-        return hubert_model
+            from fairseq import checkpoint_utils
+            models, _, _ = checkpoint_utils.load_model_ensemble_and_task([model_path],suffix="",)
+            hubert_model = models[0]
+            hubert_model = hubert_model.to(config.device)
+            if config.is_half:
+                hubert_model = hubert_model.half()
+            else:
+                hubert_model = hubert_model.float()
+            hubert_model.eval()
+            return hubert_model
     except Exception as e:
         print(e)
         return None

vc_infer_pipeline.py

@@ -11,9 +11,8 @@ from .lib.model_utils import load_hubert, change_rms
 from .pitch_extraction import FeatureExtractor
 
 from .lib.audio import MAX_INT16, load_input_audio, remix_audio
-from .lib import BASE_MODELS_DIR
 from .config import config
-from .lib.utils import gc_collect, get_filenames
+from .lib.utils import gc_collect
 
 # torchcrepe = lazyload("torchcrepe")  # Fork Feature. Crepe algo for training and preprocess
 # torch = lazyload("torch")
@@ -54,10 +53,10 @@ class VC(FeatureExtractor):
             "padding_mask": padding_mask,
             "output_layer": 9 if version == "v1" else 12,
         }
-        
-        with torch.no_grad():
-            logits = model.extract_features(**inputs)
-            feats = model.final_proj(logits[0]) if version == "v1" else logits[0]
+        feats = model.extract_features(version=version,**inputs)
+        # with torch.no_grad():
+        #     logits = model.extract_features(**inputs)
+        #     feats = model.final_proj(logits[0]) if version == "v1" else logits[0]
         if protect < 0.5 and pitch is not None and pitchf is not None:
             feats0 = feats.clone()
         if index is not None and big_npy is not None and index_rate > 0: