Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-12-25 13:58:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Handle leading spaces properly in ExLllama

Browse Source
This commit is contained in:
oobabooga 2023-06-17 19:32:04 -03:00
parent 1e97aaac95
commit c5641b65d3
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
modules/exllama.py
View File

@ -66,9 +66,17 @@ class ExllamaModel:
ids = self.generator.tokenizer.encode(prompt)
self.generator.gen_begin_reuse(ids)
initial_len = self.generator.sequence[0].shape[0]
for _ in range(state['max_new_tokens']):
has_leading_space = False
for i in range(state['max_new_tokens']):
token = self.generator.gen_single_token()
yield (self.generator.tokenizer.decode(self.generator.sequence[0][initial_len:]))
if i == 0 and self.generator.tokenizer.tokenizer.IdToPiece(int(token)).startswith('▁'):
has_leading_space = True
decoded_text = self.generator.tokenizer.decode(self.generator.sequence[0][initial_len:])
if has_leading_space:
decoded_text = ' ' + decoded_text
yield decoded_text
if token.item() == self.generator.tokenizer.eos_token_id or shared.stop_everything:
break