Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix UnicodeDecodeError for BPE-based Models (especially GLM-4) (#6357)

Browse Source
This commit is contained in:
GralchemOz 2024-09-03 10:00:59 +08:00 committed by GitHub
parent 41a8eb4eeb
commit 4c74c7a116
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
modules/text_generation.py
View File

@ -274,7 +274,12 @@ def get_reply_from_output_ids(output_ids, state=None, starting_from=0):
if (hasattr(shared.tokenizer, 'convert_ids_to_tokens') and len(output_ids) > starting_from) and not reply.startswith(' '): if (hasattr(shared.tokenizer, 'convert_ids_to_tokens') and len(output_ids) > starting_from) and not reply.startswith(' '):
first_token = shared.tokenizer.convert_ids_to_tokens(int(output_ids[starting_from])) first_token = shared.tokenizer.convert_ids_to_tokens(int(output_ids[starting_from]))
if isinstance(first_token, (bytes,)): if isinstance(first_token, (bytes,)):
first_token = first_token.decode('utf8') #try to decode the bytes to a string
try:
first_token = first_token.decode('utf8')
#if it fails, which means it's not a string in this turn, just ignore it
except UnicodeDecodeError:
first_token = ''
if first_token.startswith('▁'): if first_token.startswith('▁'):
reply = ' ' + reply reply = ' ' + reply