From 1576227f169f6b5dc7676348b75a3cc472fe2ba2 Mon Sep 17 00:00:00 2001 From: Forkoz <59298527+Ph0rk0z@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:51:01 -0500 Subject: [PATCH] Fix GGUFs with no BOS token present, mainly qwen2 models. (#6119) --------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- modules/models_settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/models_settings.py b/modules/models_settings.py index c3712db2..e9645c96 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -64,7 +64,11 @@ def get_model_metadata(model): if 'tokenizer.chat_template' in metadata: template = metadata['tokenizer.chat_template'] eos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.eos_token_id']] - bos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.bos_token_id']] + if 'tokenizer.ggml.bos_token_id' in metadata: + bos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.bos_token_id']] + else: + bos_token = "" + template = template.replace('eos_token', "'{}'".format(eos_token)) template = template.replace('bos_token', "'{}'".format(bos_token))