Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fallback to latin on decode failure

Browse Source
This commit is contained in:
nclok1405 2024-05-28 00:45:06 +09:00
parent 39db799bf1
commit f5ca611569
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
modules/llamacpp_model.py
View File

@ -114,7 +114,18 @@ class LlamaCppModel:
return self.model.tokenize(string)
def decode(self, ids, **kwargs):
return self.model.detokenize(ids).decode('utf-8', 'replace')
detokenized = self.model.detokenize(ids)
s = ""
try:
s = detokenized.decode('utf-8', 'strict')
except UnicodeDecodeError:
try:
logger.warning("Detokenized result can't be decoded in utf-8 charset. Fallback to latin.")
s = detokenized.decode('latin', 'strict')
except UnicodeDecodeError as e:
logger.warning("Detokenized result can't be decoded in latin charset. Will attempt to decode as utf-8 with invalid characters replaced with '?' (U+FFFD).")
s = detokenized.decode('utf-8', 'replace')
return s
def get_logits(self, tokens):
self.model.reset()