Mirrors/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-28 04:47:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

spm : fix special_add_bos default

Browse Source 
When this is not set in HF `tokenizer_config.json`, it should default to
true.
This commit is contained in:
Jared Van Bortel 2024-04-04 17:54:46 -04:00
parent 45983e3a47
commit d1a1b614cd
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llama.cpp
View File

@ -11562,7 +11562,7 @@ static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab &
// tokenizer.encode('', add_special_tokens=True) returns [1] // tokenizer.encode('', add_special_tokens=True) returns [1]
// tokenizer.encode('', add_special_tokens=False) returns [] // tokenizer.encode('', add_special_tokens=False) returns []
if (add_special && vocab.special_add_bos == 1) { if (add_special && vocab.special_add_bos != 0) {
GGML_ASSERT(vocab.special_bos_id != -1); GGML_ASSERT(vocab.special_bos_id != -1);
output.push_back(vocab.special_bos_id); output.push_back(vocab.special_bos_id);
} }