47 lines
1.3 KiB
Python
47 lines
1.3 KiB
Python
import os
|
|
import sys
|
|
from typing import Any, Dict
|
|
import torch
|
|
import sentencepiece as spm
|
|
|
|
PROJECT_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
|
MODULE_ROOT = os.path.join(PROJECT_ROOT, "modules")
|
|
sys.path.append(MODULE_ROOT)
|
|
from pytorch_wrapper.utils import get_model_full_path
|
|
|
|
class SpLoadModel:
|
|
"""
|
|
A wrapper class for loading a SentencePiece tokenization model.
|
|
|
|
category: PyTorch wrapper - Training
|
|
"""
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls) -> Dict[str, Any]:
|
|
"""
|
|
Defines the input types for the function.
|
|
|
|
Returns:
|
|
Dict[str, Any]: A dictionary specifying required input types.
|
|
"""
|
|
return {
|
|
"required": {
|
|
"model_path": ("STRING", {"default": "spiece.model"})
|
|
}
|
|
}
|
|
|
|
RETURN_NAMES: tuple = ("spmodel",)
|
|
RETURN_TYPES: tuple = ("SPMODEL",)
|
|
FUNCTION: str = "f"
|
|
CATEGORY: str = "Data Analysis"
|
|
|
|
@classmethod
|
|
def IS_CHANGED(cls, **kw):
|
|
return float("NaN")
|
|
|
|
def f(self, model_path: str):
|
|
model_full_path = get_model_full_path(model_path,
|
|
sub_model_dir_name="sentencepiece")
|
|
sp = spm.SentencePieceProcessor(model_file=model_full_path)
|
|
|
|
return (sp,) |