From db1da9f98d64b6d2c1d3171f56f82bc305a1eaf3 Mon Sep 17 00:00:00 2001 From: lmg-anon <139719567+lmg-anon@users.noreply.github.com> Date: Mon, 22 Jan 2024 08:07:42 -0300 Subject: [PATCH 1/3] Fix logprobs tokens in OpenAI API (#5339) --- extensions/openai/completions.py | 4 ++-- modules/text_generation.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py index c5ef36e6..b1bb0fff 100644 --- a/extensions/openai/completions.py +++ b/extensions/openai/completions.py @@ -22,7 +22,7 @@ from modules.chat import ( load_instruction_template_memoized ) from modules.presets import load_preset_memoized -from modules.text_generation import decode, encode, generate_reply +from modules.text_generation import decode, encode, generate_reply, get_reply_from_output_ids class LogitsBiasProcessor(LogitsProcessor): @@ -56,7 +56,7 @@ class LogprobProcessor(LogitsProcessor): if self.logprobs is not None: # 0-5 log_e_probabilities = F.log_softmax(logits, dim=1) top_values, top_indices = torch.topk(log_e_probabilities, k=self.logprobs + 1) - top_tokens = [decode(tok) for tok in top_indices[0]] + top_tokens = [get_reply_from_output_ids([tok]) for tok in top_indices[0]] top_probs = [float(x) for x in top_values[0]] self.token_alternatives = dict(zip(top_tokens, top_probs)) debug_msg(repr(self)) diff --git a/modules/text_generation.py b/modules/text_generation.py index c8562450..f4849840 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -268,8 +268,8 @@ def apply_stopping_strings(reply, all_stop_strings): return reply, stop_found -def get_reply_from_output_ids(output_ids, state, starting_from=0): - reply = decode(output_ids[starting_from:], state['skip_special_tokens']) +def get_reply_from_output_ids(output_ids, state=None, starting_from=0): + reply = decode(output_ids[starting_from:], state['skip_special_tokens'] if state else True) # Handle tokenizers that do not add the leading space for the first token if (hasattr(shared.tokenizer, 'convert_ids_to_tokens') and len(output_ids) > starting_from) and not reply.startswith(' '): From 166fdf09f3000a9174c3176afa86895a7c2085f0 Mon Sep 17 00:00:00 2001 From: Ercan <62525303+ercanozer@users.noreply.github.com> Date: Mon, 22 Jan 2024 06:08:51 -0500 Subject: [PATCH 2/3] API: Properly handle Images with RGBA color format (#5332) --- extensions/openai/completions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py index b1bb0fff..35dd5d14 100644 --- a/extensions/openai/completions.py +++ b/extensions/openai/completions.py @@ -156,8 +156,9 @@ def convert_history(history): img = Image.open(BytesIO(my_res.content)) except Exception: raise 'Image cannot be loaded from the URL!' - buffered = BytesIO() + if img.mode in ("RGBA", "P"): + img = img.convert("RGB") img.save(buffered, format="JPEG") img_str = base64.b64encode(buffered.getvalue()).decode('utf-8') content = f'' From fbf8ae39f84d1958db97676ddb1268b92c9df356 Mon Sep 17 00:00:00 2001 From: Cohee <18619528+Cohee1207@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:10:26 +0200 Subject: [PATCH 3/3] API: Allow content arrays for multimodal OpenAI requests (#5277) --- extensions/openai/completions.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py index 35dd5d14..ce9cc102 100644 --- a/extensions/openai/completions.py +++ b/extensions/openai/completions.py @@ -144,6 +144,26 @@ def convert_history(history): user_input = "" system_message = "" + if any(isinstance(entry['content'], list) for entry in history): + new_history = [] + for entry in history: + if isinstance(entry['content'], list): + image_url = None + content = None + for item in entry['content']: + if not isinstance(item, dict): + continue + if item['type'] == 'image_url' and isinstance(item['image_url'], dict): + image_url = item['image_url']['url'] + elif item['type'] == 'text' and isinstance(item['text'], str): + content = item['text'] + if image_url and content: + new_history.append({"image_url": image_url, "role": "user"}) + new_history.append({"content": content, "role": "user"}) + else: + new_history.append(entry) + history = new_history + for entry in history: if "image_url" in entry: image_url = entry['image_url']