From e3da126f2a86706d0ad718057e0371b9dab4bc52 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 21 Aug 2023 16:41:27 +0300 Subject: [PATCH] main : inject reverse prompt after EOS + update examples/chat.sh --- examples/chat.sh | 8 +++++--- examples/main/main.cpp | 7 ++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/chat.sh b/examples/chat.sh index 9a928ef05..bd48c8303 100755 --- a/examples/chat.sh +++ b/examples/chat.sh @@ -11,6 +11,8 @@ cd .. # # "--keep 48" is based on the contents of prompts/chat-with-bob.txt # -./main -m ./models/7B/ggml-model-q4_0.bin -c 512 -b 1024 -n 256 --keep 48 \ - --repeat_penalty 1.0 --color -i \ - -r "User:" -f prompts/chat-with-bob.txt +./main -m ./models/7B/ggml-model-q4_0.bin -c 512 -b 1024 -n -1 --keep 48 \ + --repeat_penalty 1.0 --color \ + -i --interactive-first \ + -r "User:" --in-prefix " " \ + -f prompts/chat-with-bob.txt diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 26ec48ed5..8a87b534b 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -634,9 +634,14 @@ int main(int argc, char ** argv) { llama_grammar_accept_token(ctx, grammar, id); } - // replace end of text token with newline token when in interactive mode + // replace end of text token with newline token and inject reverse prompt when in interactive mode if (id == llama_token_eos() && params.interactive && !params.instruct && !params.input_prefix_bos) { id = llama_token_nl(); + if (params.antiprompt.size() != 0) { + // tokenize and inject first reverse prompt + const auto first_antiprompt = ::llama_tokenize(ctx, params.antiprompt.front(), false); + embd_inp.insert(embd_inp.end(), first_antiprompt.begin(), first_antiprompt.end()); + } } last_n_tokens.erase(last_n_tokens.begin());