From bae14c8f13c4ff66b0d266ad1ea44ca97413258f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 7 Mar 2024 08:50:24 -0800 Subject: [PATCH] Right-truncate long chat completion prompts instead of left-truncating Instructions are usually at the beginning of the prompt. --- modules/chat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index 237e8275..a1fcb6b0 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -197,16 +197,16 @@ def generate_chat_prompt(user_input, state, **kwargs): while right - left > 1: mid = (left + right) // 2 - messages[-1]['content'] = user_message[mid:] + messages[-1]['content'] = user_message[:mid] prompt = make_prompt(messages) encoded_length = get_encoded_length(prompt) if encoded_length <= max_length: - right = mid - else: left = mid + else: + right = mid - messages[-1]['content'] = user_message[right:] + messages[-1]['content'] = user_message[:left] prompt = make_prompt(messages) encoded_length = get_encoded_length(prompt) if encoded_length > max_length: