From c7d92e6dfec3f54849f3a0ba373054d29f321ea2 Mon Sep 17 00:00:00 2001 From: Tim Miller Date: Sun, 27 Aug 2023 03:27:07 +0900 Subject: [PATCH] llama : use Unicode Escape Sequence to replace encoded characters (#2814) The use of special characters within source files can break compiling on some computers with different region and language settings. Using Unicode escape sequences should allow for the code to be compiled on all setups without needing to change your computers settings or switch regions. --- llama.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama.cpp b/llama.cpp index 62889b3ed..05c54c213 100644 --- a/llama.cpp +++ b/llama.cpp @@ -955,10 +955,10 @@ struct llama_vocab { id linefeed_id = 13; int find_bpe_rank(std::string token_left, std::string token_right) const { - replace_all(token_left, " ", "Ġ"); - replace_all(token_left, "\n", "Ċ"); - replace_all(token_right, " ", "Ġ"); - replace_all(token_right, "\n", "Ċ"); + replace_all(token_left, " ", "\u0120"); + replace_all(token_left, "\n", "\u010A"); + replace_all(token_right, " ", "\u0120"); + replace_all(token_right, "\n", "\u010A"); auto it = bpe_ranks.find(std::make_pair(token_left, token_right)); if (it == bpe_ranks.end()) {