diff --git a/src/llama.cpp b/src/llama.cpp index c30d0adfe..b652762d2 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -15440,8 +15440,8 @@ struct llm_tokenizer_bpe { break; case LLAMA_VOCAB_PRE_TYPE_VIKING: regex_exprs = { - "\\p{N}", " ?[^(\\s|.,!?…。,、।۔،)]+", + "\\p{N}", }; break; default: