nodes/Zuellni-ComfyUI-ExLlama-Nodes
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert autosplit

Browse Source
This commit is contained in:
Zuellni
2023-11-22 16:04:01 +01:00
parent 996fdd012d
commit 51bb1bcf84
1 changed files with 18 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
exllama.py
+18 -23
View File
@@ -34,7 +34,6 @@ class Loader:
self.generator = None self.generator = None
self.gpu_split = None self.gpu_split = None
self.cache_8bit = False self.cache_8bit = False
self.cache_lazy = False
def process(self, model_dir, gpu_split, cache_8bit, max_seq_len): def process(self, model_dir, gpu_split, cache_8bit, max_seq_len):
self.unload() self.unload()
@@ -42,10 +41,10 @@ class Loader:
self.config.model_dir = model_dir self.config.model_dir = model_dir
self.config.prepare() self.config.prepare()
if gpu_split == "auto": if gpu_split:
self.cache_lazy = True
elif gpu_split:
self.gpu_split = [float(a) for a in gpu_split.split(",")] self.gpu_split = [float(a) for a in gpu_split.split(",")]
else:
self.gpu_split = None
if max_seq_len: if max_seq_len:
self.config.max_seq_len = max_seq_len self.config.max_seq_len = max_seq_len
@@ -60,16 +59,12 @@ class Loader:
return return
self.base = ExLlamaV2(self.config) self.base = ExLlamaV2(self.config)
self.base.load(gpu_split=self.gpu_split)
if self.cache_8bit: if self.cache_8bit:
self.cache = ExLlamaV2Cache_8bit(self.base, lazy=self.cache_lazy) self.cache = ExLlamaV2Cache_8bit(self.base)
else: else:
self.cache = ExLlamaV2Cache(self.base, lazy=self.cache_lazy) self.cache = ExLlamaV2Cache(self.base)
if self.cache_lazy:
self.base.load_autosplit(self.cache)
else:
self.base.load(gpu_split=self.gpu_split)
self.tokenizer = ExLlamaV2Tokenizer(self.config) self.tokenizer = ExLlamaV2Tokenizer(self.config)
self.generator = ExLlamaV2StreamingGenerator(self.base, self.cache, self.tokenizer) self.generator = ExLlamaV2StreamingGenerator(self.base, self.cache, self.tokenizer)
@@ -92,14 +87,14 @@ class Generator:
"model": ("EXL_MODEL",), "model": ("EXL_MODEL",),
"unload": ("BOOLEAN", {"default": False}), "unload": ("BOOLEAN", {"default": False}),
"single_line": ("BOOLEAN", {"default": False}), "single_line": ("BOOLEAN", {"default": False}),
"temperature_last": ("BOOLEAN", {"default": True}),
"max_tokens": ("INT", {"default": 128, "max": 2**16}), "max_tokens": ("INT", {"default": 128, "max": 2**16}),
"temperature": ("FLOAT", {"default": 1, "max": 2, "step": 0.01}), "temperature": ("FLOAT", {"default": 1, "max": 2, "step": 0.01}),
"temperature_last": ("BOOLEAN", {"default": True}), "top_k": ("INT", {"max": 200}),
"min_p": ("FLOAT", {"default": 0.1, "max": 1, "step": 0.01}), "min_p": ("FLOAT", {"default": 0.1, "max": 1, "step": 0.01}),
"top_p": ("FLOAT", {"default": 1, "max": 1, "step": 0.01}), "top_p": ("FLOAT", {"default": 1, "max": 1, "step": 0.01}),
"typical_p": ("FLOAT", {"default": 1, "max": 1, "step": 0.01}), "typical": ("FLOAT", {"default": 1, "max": 1, "step": 0.01}),
"top_k": ("INT", {"max": 200}), "penalty": ("FLOAT", {"default": 1, "min": 1, "max": 2, "step": 0.01}),
"rep_penalty": ("FLOAT", {"default": 1, "min": 1, "max": 2, "step": 0.01}),
"seed": ("INT", {"max": 2**64 - 1}), "seed": ("INT", {"max": 2**64 - 1}),
"text": ("STRING", {"multiline": True}), "text": ("STRING", {"multiline": True}),
}, },
@@ -119,14 +114,14 @@ class Generator:
model, model,
unload, unload,
single_line, single_line,
temperature_last,
max_tokens, max_tokens,
temperature, temperature,
temperature_last, top_k,
min_p, min_p,
top_p, top_p,
typical_p, typical,
top_k, penalty,
rep_penalty,
seed, seed,
text, text,
info=None, info=None,
@@ -152,13 +147,13 @@ class Generator:
random.seed(seed) random.seed(seed)
settings = ExLlamaV2Sampler.Settings() settings = ExLlamaV2Sampler.Settings()
settings.temperature = temperature
settings.temperature_last = temperature_last settings.temperature_last = temperature_last
settings.temperature = temperature
settings.top_k = top_k
settings.min_p = min_p settings.min_p = min_p
settings.top_p = top_p settings.top_p = top_p
settings.typical = typical_p settings.typical = typical
settings.top_k = top_k settings.token_repetition_penalty = penalty
settings.token_repetition_penalty = rep_penalty
model.generator.begin_stream(input, settings, token_healing=True) model.generator.begin_stream(input, settings, token_healing=True)
progress = ProgressBar(max_tokens) progress = ProgressBar(max_tokens)