From ab50f80542788dd7fa21b20ac91fddc8c9766c23 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 8 Mar 2023 02:46:35 -0300 Subject: [PATCH 01/20] New text streaming method (much faster) --- modules/callbacks.py | 75 ++++++++++++++++++++++++++++++++++++ modules/stopping_criteria.py | 32 --------------- modules/text_generation.py | 66 +++++++++++++++++++++++-------- server.py | 3 -- 4 files changed, 124 insertions(+), 52 deletions(-) create mode 100644 modules/callbacks.py delete mode 100644 modules/stopping_criteria.py diff --git a/modules/callbacks.py b/modules/callbacks.py new file mode 100644 index 00000000..15674b8a --- /dev/null +++ b/modules/callbacks.py @@ -0,0 +1,75 @@ +from queue import Queue +from threading import Thread + +import torch +import transformers + +import modules.shared as shared + + +# Copied from https://github.com/PygmalionAI/gradio-ui/ +class _SentinelTokenStoppingCriteria(transformers.StoppingCriteria): + + def __init__(self, sentinel_token_ids: torch.LongTensor, + starting_idx: int): + transformers.StoppingCriteria.__init__(self) + self.sentinel_token_ids = sentinel_token_ids + self.starting_idx = starting_idx + + def __call__(self, input_ids: torch.LongTensor, + _scores: torch.FloatTensor) -> bool: + for sample in input_ids: + trimmed_sample = sample[self.starting_idx:] + # Can't unfold, output is still too tiny. Skip. + if trimmed_sample.shape[-1] < self.sentinel_token_ids.shape[-1]: + continue + + for window in trimmed_sample.unfold( + 0, self.sentinel_token_ids.shape[-1], 1): + if torch.all(torch.eq(self.sentinel_token_ids, window)): + return True + return False + +class Stream(transformers.StoppingCriteria): + def __init__(self, callback_func=None): + self.callback_func = callback_func + + def __call__(self, input_ids, scores) -> bool: + if self.callback_func is not None: + self.callback_func(input_ids[0]) + return False + +class Iteratorize: + + """ + Transforms a function that takes a callback + into a lazy iterator (generator). + """ + + def __init__(self, func, kwargs={}, callback=None): + self.mfunc=func + self.c_callback=callback + self.q = Queue(maxsize=1) + self.sentinel = object() + self.kwargs = kwargs + + def _callback(val): + self.q.put(val) + + def gentask(): + ret = self.mfunc(callback=_callback, **self.kwargs) + self.q.put(self.sentinel) + if self.c_callback: + self.c_callback(ret) + + Thread(target=gentask).start() + + def __iter__(self): + return self + + def __next__(self): + obj = self.q.get(True,None) + if obj is self.sentinel: + raise StopIteration + else: + return obj diff --git a/modules/stopping_criteria.py b/modules/stopping_criteria.py deleted file mode 100644 index 44a631b3..00000000 --- a/modules/stopping_criteria.py +++ /dev/null @@ -1,32 +0,0 @@ -''' -This code was copied from - -https://github.com/PygmalionAI/gradio-ui/ - -''' - -import torch -import transformers - - -class _SentinelTokenStoppingCriteria(transformers.StoppingCriteria): - - def __init__(self, sentinel_token_ids: torch.LongTensor, - starting_idx: int): - transformers.StoppingCriteria.__init__(self) - self.sentinel_token_ids = sentinel_token_ids - self.starting_idx = starting_idx - - def __call__(self, input_ids: torch.LongTensor, - _scores: torch.FloatTensor) -> bool: - for sample in input_ids: - trimmed_sample = sample[self.starting_idx:] - # Can't unfold, output is still too tiny. Skip. - if trimmed_sample.shape[-1] < self.sentinel_token_ids.shape[-1]: - continue - - for window in trimmed_sample.unfold( - 0, self.sentinel_token_ids.shape[-1], 1): - if torch.all(torch.eq(self.sentinel_token_ids, window)): - return True - return False diff --git a/modules/text_generation.py b/modules/text_generation.py index 4af53273..436afbeb 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -5,13 +5,13 @@ import time import numpy as np import torch import transformers -from tqdm import tqdm import modules.shared as shared +from modules.callbacks import (Iteratorize, Stream, + _SentinelTokenStoppingCriteria) from modules.extensions import apply_extensions from modules.html_generator import generate_4chan_html, generate_basic_html from modules.models import local_rank -from modules.stopping_criteria import _SentinelTokenStoppingCriteria def get_max_prompt_length(tokens): @@ -103,7 +103,9 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi yield formatted_outputs(reply, shared.model_name) t1 = time.time() - print(f"Output generated in {(t1-t0):.2f} seconds.") + output = encode(reply)[0] + input_ids = encode(question) + print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output)-len(input_ids[0]))/(t1-t0):.2f} tokens/s, {len(output)-len(input_ids[0])} tokens)") return original_question = question @@ -113,6 +115,7 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi print(f"\n\n{question}\n--------------------\n") input_ids = encode(question, max_new_tokens) + original_input_ids = input_ids cuda = "" if (shared.args.cpu or shared.args.deepspeed or shared.args.flexgen) else ".cuda()" n = shared.tokenizer.eos_token_id if eos_token is None else int(encode(eos_token)[0][-1]) if stopping_string is not None: @@ -126,10 +129,11 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi ) ]) else: - stopping_criteria_list = None + stopping_criteria_list = [] if not shared.args.flexgen: generate_params = [ + f"max_new_tokens=max_new_tokens", f"eos_token_id={n}", f"stopping_criteria=stopping_criteria_list", f"do_sample={do_sample}", @@ -147,24 +151,21 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi ] else: generate_params = [ + f"max_new_tokens={max_new_tokens if shared.args.no_stream else 8}", f"do_sample={do_sample}", f"temperature={temperature}", f"stop={n}", ] if shared.args.deepspeed: generate_params.append("synced_gpus=True") - if shared.args.no_stream: - generate_params.append("max_new_tokens=max_new_tokens") - else: - generate_params.append("max_new_tokens=8") if shared.soft_prompt: inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids) generate_params.insert(0, "inputs_embeds=inputs_embeds") - generate_params.insert(0, "filler_input_ids") + generate_params.insert(0, "inputs=filler_input_ids") else: - generate_params.insert(0, "input_ids") + generate_params.insert(0, "inputs=input_ids") - # Generate the entire reply at once + # Generate the entire reply at once. if shared.args.no_stream: with torch.no_grad(): output = eval(f"shared.model.generate({', '.join(generate_params)}){cuda}")[0] @@ -175,18 +176,45 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi if not (shared.args.chat or shared.args.cai_chat): reply = original_question + apply_extensions(reply[len(question):], "output") - t1 = time.time() - print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output)-len(input_ids[0]))/(t1-t0)/8:.2f} it/s, {len(output)-len(input_ids[0])} tokens)") yield formatted_outputs(reply, shared.model_name) - # Generate the reply 8 tokens at a time - else: + # Stream the reply 1 token at a time. + # This is based on the trick of using 'stopping_criteria' to create an iterator. + elif not shared.args.flexgen: + + def generate_with_callback(callback=None, **kwargs): + if 'stopping_criteria' not in kwargs: + kwargs['stopping_criteria'] = [] + kwargs['stopping_criteria'].append(Stream(callback_func=callback)) + shared.model.generate(**kwargs)[0] + + def generate_with_streaming(**kwargs): + return Iteratorize(generate_with_callback, kwargs, callback=None) + yield formatted_outputs(original_question, shared.model_name) - for i in tqdm(range(max_new_tokens//8+1)): + for output in eval(f"generate_with_streaming({', '.join(generate_params)})"): + if shared.soft_prompt: + output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) + + reply = decode(output) + if not (shared.args.chat or shared.args.cai_chat): + reply = original_question + apply_extensions(reply[len(question):], "output") + yield formatted_outputs(reply, shared.model_name) + + if not shared.args.flexgen: + if output[-1] == n: + break + else: + if np.count_nonzero(input_ids[0] == n) < np.count_nonzero(output == n): + break + + # Stream the output naively for FlexGen since it doesn't support 'stopping_criteria' + else: + for i in range(max_new_tokens//8+1): clear_torch_cache() with torch.no_grad(): - output = eval(f"shared.model.generate({', '.join(generate_params)}){cuda}")[0] + output = eval(f"shared.model.generate({', '.join(generate_params)})")[0] if shared.soft_prompt: output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) @@ -206,3 +234,7 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi if shared.soft_prompt: inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids) + + t1 = time.time() + print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output)-len(original_input_ids[0]))/(t1-t0):.2f} tokens/s, {len(output)-len(original_input_ids[0])} tokens)") + return diff --git a/server.py b/server.py index 9f584ba3..42897b0b 100644 --- a/server.py +++ b/server.py @@ -18,9 +18,6 @@ from modules.html_generator import generate_chat_html from modules.models import load_model, load_soft_prompt from modules.text_generation import generate_reply -if (shared.args.chat or shared.args.cai_chat) and not shared.args.no_stream: - print('Warning: chat mode currently becomes somewhat slower with text streaming on.\nConsider starting the web UI with the --no-stream option.\n') - # Loading custom settings settings_file = None if shared.args.settings is not None and Path(shared.args.settings).exists(): From 0e16c0bacb88ad0f5420fd2aa2c6cfadf38e2579 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 8 Mar 2023 02:50:49 -0300 Subject: [PATCH 02/20] Remove redeclaration of a function --- modules/RWKV.py | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/modules/RWKV.py b/modules/RWKV.py index b226a195..70deab28 100644 --- a/modules/RWKV.py +++ b/modules/RWKV.py @@ -7,6 +7,7 @@ import numpy as np from tokenizers import Tokenizer import modules.shared as shared +from modules.callbacks import Iteratorize np.set_printoptions(precision=4, suppress=True, linewidth=200) @@ -73,38 +74,3 @@ class RWKVTokenizer: def decode(self, ids): return self.tokenizer.decode(ids) - -class Iteratorize: - - """ - Transforms a function that takes a callback - into a lazy iterator (generator). - """ - - def __init__(self, func, kwargs={}, callback=None): - self.mfunc=func - self.c_callback=callback - self.q = Queue(maxsize=1) - self.sentinel = object() - self.kwargs = kwargs - - def _callback(val): - self.q.put(val) - - def gentask(): - ret = self.mfunc(callback=_callback, **self.kwargs) - self.q.put(self.sentinel) - if self.c_callback: - self.c_callback(ret) - - Thread(target=gentask).start() - - def __iter__(self): - return self - - def __next__(self): - obj = self.q.get(True,None) - if obj is self.sentinel: - raise StopIteration - else: - return obj From 72d539dbff6f946fbbd1d8806361dccbc241f8ec Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 8 Mar 2023 02:54:47 -0300 Subject: [PATCH 03/20] Better separate the FlexGen case --- modules/text_generation.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index 436afbeb..a8157a76 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -201,12 +201,8 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi reply = original_question + apply_extensions(reply[len(question):], "output") yield formatted_outputs(reply, shared.model_name) - if not shared.args.flexgen: - if output[-1] == n: - break - else: - if np.count_nonzero(input_ids[0] == n) < np.count_nonzero(output == n): - break + if output[-1] == n: + break # Stream the output naively for FlexGen since it doesn't support 'stopping_criteria' else: @@ -223,14 +219,9 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi reply = original_question + apply_extensions(reply[len(question):], "output") yield formatted_outputs(reply, shared.model_name) - if not shared.args.flexgen: - if output[-1] == n: - break - input_ids = torch.reshape(output, (1, output.shape[0])) - else: - if np.count_nonzero(input_ids[0] == n) < np.count_nonzero(output == n): - break - input_ids = np.reshape(output, (1, output.shape[0])) + if np.count_nonzero(input_ids[0] == n) < np.count_nonzero(output == n): + break + input_ids = np.reshape(output, (1, output.shape[0])) if shared.soft_prompt: inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids) From ad2970374adeb58aec1d7748b02a8c82cc524c0a Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 8 Mar 2023 03:00:06 -0300 Subject: [PATCH 04/20] Readability improvements --- modules/text_generation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index a8157a76..9477fe41 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -195,8 +195,8 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi for output in eval(f"generate_with_streaming({', '.join(generate_params)})"): if shared.soft_prompt: output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) - reply = decode(output) + if not (shared.args.chat or shared.args.cai_chat): reply = original_question + apply_extensions(reply[len(question):], "output") yield formatted_outputs(reply, shared.model_name) @@ -213,16 +213,16 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi output = eval(f"shared.model.generate({', '.join(generate_params)})")[0] if shared.soft_prompt: output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) - reply = decode(output) + if not (shared.args.chat or shared.args.cai_chat): reply = original_question + apply_extensions(reply[len(question):], "output") yield formatted_outputs(reply, shared.model_name) if np.count_nonzero(input_ids[0] == n) < np.count_nonzero(output == n): break - input_ids = np.reshape(output, (1, output.shape[0])) + input_ids = np.reshape(output, (1, output.shape[0])) if shared.soft_prompt: inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids) From 33fb6aed74ebfd50f12373fcbe2f7c0d285022d3 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 8 Mar 2023 03:08:16 -0300 Subject: [PATCH 05/20] Minor bug fix --- modules/text_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index 9477fe41..35617314 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -115,7 +115,7 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi print(f"\n\n{question}\n--------------------\n") input_ids = encode(question, max_new_tokens) - original_input_ids = input_ids + original_input_ids = output = input_ids cuda = "" if (shared.args.cpu or shared.args.deepspeed or shared.args.flexgen) else ".cuda()" n = shared.tokenizer.eos_token_id if eos_token is None else int(encode(eos_token)[0][-1]) if stopping_string is not None: From add9330e5e90e33f3f8bbe0ea42290475deb9998 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 8 Mar 2023 11:26:29 -0300 Subject: [PATCH 06/20] Bug fixes --- modules/text_generation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index 35617314..8f5ea798 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -115,7 +115,8 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi print(f"\n\n{question}\n--------------------\n") input_ids = encode(question, max_new_tokens) - original_input_ids = output = input_ids + original_input_ids = input_ids + output = input_ids[0] cuda = "" if (shared.args.cpu or shared.args.deepspeed or shared.args.flexgen) else ".cuda()" n = shared.tokenizer.eos_token_id if eos_token is None else int(encode(eos_token)[0][-1]) if stopping_string is not None: @@ -186,7 +187,8 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi if 'stopping_criteria' not in kwargs: kwargs['stopping_criteria'] = [] kwargs['stopping_criteria'].append(Stream(callback_func=callback)) - shared.model.generate(**kwargs)[0] + clear_torch_cache() + shared.model.generate(**kwargs) def generate_with_streaming(**kwargs): return Iteratorize(generate_with_callback, kwargs, callback=None) @@ -208,7 +210,6 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi else: for i in range(max_new_tokens//8+1): clear_torch_cache() - with torch.no_grad(): output = eval(f"shared.model.generate({', '.join(generate_params)})")[0] if shared.soft_prompt: From 59b5f7a4b731c528f0fa53d70eb3318d3a1727df Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 8 Mar 2023 12:13:40 -0300 Subject: [PATCH 07/20] Improve usage of stopping_criteria --- modules/text_generation.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index 8f5ea798..6a59f9a7 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -119,18 +119,11 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi output = input_ids[0] cuda = "" if (shared.args.cpu or shared.args.deepspeed or shared.args.flexgen) else ".cuda()" n = shared.tokenizer.eos_token_id if eos_token is None else int(encode(eos_token)[0][-1]) + stopping_criteria_list = transformers.StoppingCriteriaList() if stopping_string is not None: - # The stopping_criteria code below was copied from - # https://github.com/PygmalionAI/gradio-ui/blob/master/src/model.py + # Copied from https://github.com/PygmalionAI/gradio-ui/blob/master/src/model.py t = encode(stopping_string, 0, add_special_tokens=False) - stopping_criteria_list = transformers.StoppingCriteriaList([ - _SentinelTokenStoppingCriteria( - sentinel_token_ids=t, - starting_idx=len(input_ids[0]) - ) - ]) - else: - stopping_criteria_list = [] + stopping_criteria_list.append(_SentinelTokenStoppingCriteria(sentinel_token_ids=t, starting_idx=len(input_ids[0]))) if not shared.args.flexgen: generate_params = [ @@ -184,17 +177,17 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi elif not shared.args.flexgen: def generate_with_callback(callback=None, **kwargs): - if 'stopping_criteria' not in kwargs: - kwargs['stopping_criteria'] = [] kwargs['stopping_criteria'].append(Stream(callback_func=callback)) clear_torch_cache() - shared.model.generate(**kwargs) + with torch.no_grad(): + shared.model.generate(**kwargs) def generate_with_streaming(**kwargs): return Iteratorize(generate_with_callback, kwargs, callback=None) yield formatted_outputs(original_question, shared.model_name) for output in eval(f"generate_with_streaming({', '.join(generate_params)})"): + print(print('Used vram in gib:', torch.cuda.memory_allocated() / 1024**3)) if shared.soft_prompt: output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) reply = decode(output) From 96c51973f9e551055dac2e135e9c4229cbf40ad0 Mon Sep 17 00:00:00 2001 From: Xan <70198941+xanthousm@users.noreply.github.com> Date: Sat, 11 Mar 2023 22:50:59 +1100 Subject: [PATCH 08/20] --auto-launch and "Is typing..." - Added `--auto-launch` arg to open web UI in the default browser when ready. - Changed chat.py to display user input immediately and "*Is typing...*" as a temporary reply while generating text. Most noticeable when using `--no-stream`. --- modules/chat.py | 3 +++ modules/shared.py | 1 + server.py | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index f40f8299..0f029fe2 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -120,6 +120,9 @@ def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical else: prompt = custom_generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size) + #display user input and "*is typing...*" imediately + yield shared.history['visible']+[[visible_text, '*Is typing...*']] + # Generate reply = '' for i in range(chat_generation_attempts): diff --git a/modules/shared.py b/modules/shared.py index 2acb047f..c42ba7ed 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -90,4 +90,5 @@ parser.add_argument('--listen', action='store_true', help='Make the web UI reach parser.add_argument('--listen-port', type=int, help='The listening port that the server will use.') parser.add_argument('--share', action='store_true', help='Create a public URL. This is useful for running the web UI on Google Colab or similar.') parser.add_argument('--verbose', action='store_true', help='Print the prompts to the terminal.') +parser.add_argument('--auto-launch', action='store_true', default=False, help='Open the web UI in the default browser upon launch') args = parser.parse_args() diff --git a/server.py b/server.py index c2977f41..ad483eb5 100644 --- a/server.py +++ b/server.py @@ -372,9 +372,9 @@ else: shared.gradio['interface'].queue() if shared.args.listen: - shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_name='0.0.0.0', server_port=shared.args.listen_port) + shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_name='0.0.0.0', server_port=shared.args.listen_port, inbrowser=shared.args.auto_launch) else: - shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_port=shared.args.listen_port) + shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_port=shared.args.listen_port, inbrowser=shared.args.auto_launch) # I think that I will need this later while True: From 2743dd736a431e54a7220ef7e29ad5d31797611c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 11 Mar 2023 10:50:18 -0300 Subject: [PATCH 09/20] Add *Is typing...* to impersonate as well --- modules/chat.py | 5 ++++- server.py | 9 +++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index 0f029fe2..5bf96b1a 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -120,7 +120,7 @@ def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical else: prompt = custom_generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size) - #display user input and "*is typing...*" imediately + # Display user input and "*is typing...*" imediately yield shared.history['visible']+[[visible_text, '*Is typing...*']] # Generate @@ -161,6 +161,9 @@ def impersonate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typ prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size, impersonate=True) + # Display "*is typing...*" imediately + yield '*Is typing...*' + reply = '' for i in range(chat_generation_attempts): for reply in generate_reply(prompt+reply, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, eos_token=eos_token, stopping_string=f"\n{name2}:"): diff --git a/server.py b/server.py index ad483eb5..c65443ec 100644 --- a/server.py +++ b/server.py @@ -272,10 +272,10 @@ if shared.args.chat or shared.args.cai_chat: function_call = 'chat.cai_chatbot_wrapper' if shared.args.cai_chat else 'chat.chatbot_wrapper' - gen_events.append(shared.gradio['Generate'].click(eval(function_call), shared.input_params, shared.gradio['display'], show_progress=shared.args.no_stream, api_name='textgen')) - gen_events.append(shared.gradio['textbox'].submit(eval(function_call), shared.input_params, shared.gradio['display'], show_progress=shared.args.no_stream)) - gen_events.append(shared.gradio['Regenerate'].click(chat.regenerate_wrapper, shared.input_params, shared.gradio['display'], show_progress=shared.args.no_stream)) - gen_events.append(shared.gradio['Impersonate'].click(chat.impersonate_wrapper, shared.input_params, shared.gradio['textbox'], show_progress=shared.args.no_stream)) + gen_events.append(shared.gradio['Generate'].click(eval(function_call), shared.input_params, shared.gradio['display'], show_progress=False, api_name='textgen')) + gen_events.append(shared.gradio['textbox'].submit(eval(function_call), shared.input_params, shared.gradio['display'], show_progress=False)) + gen_events.append(shared.gradio['Regenerate'].click(chat.regenerate_wrapper, shared.input_params, shared.gradio['display'], show_progress=False)) + gen_events.append(shared.gradio['Impersonate'].click(chat.impersonate_wrapper, shared.input_params, shared.gradio['textbox'], show_progress=False)) shared.gradio['Stop'].click(chat.stop_everything_event, [], [], cancels=gen_events) shared.gradio['Copy last reply'].click(chat.send_last_reply_to_input, [], shared.gradio['textbox'], show_progress=shared.args.no_stream) @@ -309,6 +309,7 @@ if shared.args.chat or shared.args.cai_chat: reload_inputs = [shared.gradio['name1'], shared.gradio['name2']] if shared.args.cai_chat else [] shared.gradio['upload_chat_history'].upload(reload_func, reload_inputs, [shared.gradio['display']]) shared.gradio['upload_img_me'].upload(reload_func, reload_inputs, [shared.gradio['display']]) + shared.gradio['Stop'].click(reload_func, reload_inputs, [shared.gradio['display']]) shared.gradio['interface'].load(lambda : chat.load_default_history(shared.settings[f'name1{suffix}'], shared.settings[f'name2{suffix}']), None, None) shared.gradio['interface'].load(reload_func, reload_inputs, [shared.gradio['display']], show_progress=True) From 501afbc23408df04d2d545b2100bde55b6611598 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 11 Mar 2023 14:47:30 -0300 Subject: [PATCH 10/20] Add requests to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6133f394..a7df93bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ bitsandbytes==0.37.0 flexgen==0.1.7 gradio==3.18.0 numpy +requests rwkv==0.1.0 safetensors==0.2.8 sentencepiece From 195e99d0b6d116105c0adc0978a4ec4dbb0d847c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 11 Mar 2023 16:11:15 -0300 Subject: [PATCH 11/20] Add llama_prompts extension --- extensions/llama_prompts/script.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 extensions/llama_prompts/script.py diff --git a/extensions/llama_prompts/script.py b/extensions/llama_prompts/script.py new file mode 100644 index 00000000..e45cd445 --- /dev/null +++ b/extensions/llama_prompts/script.py @@ -0,0 +1,18 @@ +import gradio as gr +import modules.shared as shared +import pandas as pd + +df = pd.read_csv("https://raw.githubusercontent.com/devbrones/llama-prompts/main/prompts/prompts.csv") + +def get_prompt_by_name(name): + if name == 'None': + return '' + else: + return df[df['Prompt name'] == name].iloc[0]['Prompt'].replace('\\n', '\n') + +def ui(): + if not shared.args.chat or share.args.cai_chat: + choices = ['None'] + list(df['Prompt name']) + + prompts_menu = gr.Dropdown(value=choices[0], choices=choices, label='Prompt') + prompts_menu.change(get_prompt_by_name, prompts_menu, shared.gradio['textbox']) From 37f0166b2d6b0f2938a5a4c1762479829de1c5be Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 11 Mar 2023 23:14:49 -0300 Subject: [PATCH 12/20] Fix memory leak in new streaming (second attempt) --- modules/callbacks.py | 5 ++++- modules/text_generation.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/callbacks.py b/modules/callbacks.py index 15674b8a..05e8fafa 100644 --- a/modules/callbacks.py +++ b/modules/callbacks.py @@ -49,7 +49,7 @@ class Iteratorize: def __init__(self, func, kwargs={}, callback=None): self.mfunc=func self.c_callback=callback - self.q = Queue(maxsize=1) + self.q = Queue() self.sentinel = object() self.kwargs = kwargs @@ -73,3 +73,6 @@ class Iteratorize: raise StopIteration else: return obj + + def __del__(self): + pass diff --git a/modules/text_generation.py b/modules/text_generation.py index 6a59f9a7..5d01c8cb 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -187,7 +187,6 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi yield formatted_outputs(original_question, shared.model_name) for output in eval(f"generate_with_streaming({', '.join(generate_params)})"): - print(print('Used vram in gib:', torch.cuda.memory_allocated() / 1024**3)) if shared.soft_prompt: output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) reply = decode(output) From 0bd54309887f6e7adc7e59d4f8675ed6f322bb81 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 12 Mar 2023 02:04:28 -0300 Subject: [PATCH 13/20] Use 'with' statement to better handle streaming memory --- modules/RWKV.py | 10 +++++----- modules/callbacks.py | 27 +++++++++++++++++++++++---- modules/text_generation.py | 19 ++++++++++--------- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/modules/RWKV.py b/modules/RWKV.py index 70deab28..836d31dc 100644 --- a/modules/RWKV.py +++ b/modules/RWKV.py @@ -50,11 +50,11 @@ class RWKVModel: return context+self.pipeline.generate(context, token_count=token_count, args=args, callback=callback) def generate_with_streaming(self, **kwargs): - iterable = Iteratorize(self.generate, kwargs, callback=None) - reply = kwargs['context'] - for token in iterable: - reply += token - yield reply + with Iteratorize(self.generate, kwargs, callback=None) as generator: + reply = kwargs['context'] + for token in generator: + reply += token + yield reply class RWKVTokenizer: def __init__(self): diff --git a/modules/callbacks.py b/modules/callbacks.py index 05e8fafa..e0d1c988 100644 --- a/modules/callbacks.py +++ b/modules/callbacks.py @@ -1,3 +1,4 @@ +import gc from queue import Queue from threading import Thread @@ -6,7 +7,6 @@ import transformers import modules.shared as shared - # Copied from https://github.com/PygmalionAI/gradio-ui/ class _SentinelTokenStoppingCriteria(transformers.StoppingCriteria): @@ -52,17 +52,24 @@ class Iteratorize: self.q = Queue() self.sentinel = object() self.kwargs = kwargs + self.stop_now = False def _callback(val): + if self.stop_now: + raise ValueError self.q.put(val) def gentask(): - ret = self.mfunc(callback=_callback, **self.kwargs) + try: + ret = self.mfunc(callback=_callback, **self.kwargs) + except ValueError: + pass self.q.put(self.sentinel) if self.c_callback: self.c_callback(ret) - Thread(target=gentask).start() + self.thread = Thread(target=gentask) + self.thread.start() def __iter__(self): return self @@ -75,4 +82,16 @@ class Iteratorize: return obj def __del__(self): - pass + clear_torch_cache() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop_now = True + clear_torch_cache() + +def clear_torch_cache(): + gc.collect() + if not shared.args.cpu: + torch.cuda.empty_cache() diff --git a/modules/text_generation.py b/modules/text_generation.py index 5d01c8cb..7f5aad5e 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -186,17 +186,18 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi return Iteratorize(generate_with_callback, kwargs, callback=None) yield formatted_outputs(original_question, shared.model_name) - for output in eval(f"generate_with_streaming({', '.join(generate_params)})"): - if shared.soft_prompt: - output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) - reply = decode(output) + with eval(f"generate_with_streaming({', '.join(generate_params)})") as generator: + for output in generator: + if shared.soft_prompt: + output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) + reply = decode(output) - if not (shared.args.chat or shared.args.cai_chat): - reply = original_question + apply_extensions(reply[len(question):], "output") - yield formatted_outputs(reply, shared.model_name) + if not (shared.args.chat or shared.args.cai_chat): + reply = original_question + apply_extensions(reply[len(question):], "output") + yield formatted_outputs(reply, shared.model_name) - if output[-1] == n: - break + if output[-1] == n: + break # Stream the output naively for FlexGen since it doesn't support 'stopping_criteria' else: From 433f6350bc794e0a904e1a34abaffe49a106a484 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 11 Mar 2023 21:21:30 -0800 Subject: [PATCH 14/20] Load and save character files in UTF-8 --- modules/chat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index f40f8299..a0cae949 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -332,7 +332,7 @@ def load_character(_character, name1, name2): shared.history['visible'] = [] if _character != 'None': shared.character = _character - data = json.loads(open(Path(f'characters/{_character}.json'), 'r').read()) + data = json.loads(open(Path(f'characters/{_character}.json'), 'r', encoding='utf-8').read()) name2 = data['char_name'] if 'char_persona' in data and data['char_persona'] != '': context += f"{data['char_name']}'s Persona: {data['char_persona']}\n" @@ -372,7 +372,7 @@ def upload_character(json_file, img, tavern=False): i += 1 if tavern: outfile_name = f'TavernAI-{outfile_name}' - with open(Path(f'characters/{outfile_name}.json'), 'w') as f: + with open(Path(f'characters/{outfile_name}.json'), 'w', encoding='utf-8') as f: f.write(json_file) if img is not None: img = Image.open(io.BytesIO(img)) From b0e8cb8c889cdadd9779517ba8055114b39357cd Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 12 Mar 2023 02:31:45 -0300 Subject: [PATCH 15/20] Various fixes in chat mode --- modules/chat.py | 16 +++--- modules/text_generation.py | 102 +++++++++++++++++++------------------ 2 files changed, 62 insertions(+), 56 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index f40f8299..69d81e94 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -115,14 +115,18 @@ def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical visible_text = visible_text.replace('\n', '
') text = apply_extensions(text, "input") - if custom_generate_chat_prompt is None: - prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size) - else: - prompt = custom_generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size) - # Generate reply = '' for i in range(chat_generation_attempts): + + # The prompt needs to be generated here because, as the reply + # grows, it may become necessary to remove more old messages to + # fit into the 2048 tokens window. + if custom_generate_chat_prompt is None: + prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size-len(encode(' '+reply)[0])) + else: + prompt = custom_generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size-len(encode(' '+reply)[0])) + for reply in generate_reply(f"{prompt}{' ' if len(reply) > 0 else ''}{reply}", max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, eos_token=eos_token, stopping_string=f"\n{name1}:"): # Extracting the reply @@ -156,10 +160,10 @@ def impersonate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typ if 'pygmalion' in shared.model_name.lower(): name1 = "You" - prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size, impersonate=True) reply = '' for i in range(chat_generation_attempts): + prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size-len(encode(' '+reply)[0]), impersonate=True) for reply in generate_reply(prompt+reply, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, eos_token=eos_token, stopping_string=f"\n{name2}:"): reply, next_character_found, substring_found = extract_message_from_reply(prompt, reply, name1, name2, check, impersonate=True) if not substring_found: diff --git a/modules/text_generation.py b/modules/text_generation.py index 7f5aad5e..2460df4f 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -159,35 +159,53 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi else: generate_params.insert(0, "inputs=input_ids") - # Generate the entire reply at once. - if shared.args.no_stream: - with torch.no_grad(): - output = eval(f"shared.model.generate({', '.join(generate_params)}){cuda}")[0] - if shared.soft_prompt: - output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) - - reply = decode(output) - if not (shared.args.chat or shared.args.cai_chat): - reply = original_question + apply_extensions(reply[len(question):], "output") - - yield formatted_outputs(reply, shared.model_name) - - # Stream the reply 1 token at a time. - # This is based on the trick of using 'stopping_criteria' to create an iterator. - elif not shared.args.flexgen: - - def generate_with_callback(callback=None, **kwargs): - kwargs['stopping_criteria'].append(Stream(callback_func=callback)) - clear_torch_cache() + try: + # Generate the entire reply at once. + if shared.args.no_stream: with torch.no_grad(): - shared.model.generate(**kwargs) + output = eval(f"shared.model.generate({', '.join(generate_params)}){cuda}")[0] + if shared.soft_prompt: + output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) - def generate_with_streaming(**kwargs): - return Iteratorize(generate_with_callback, kwargs, callback=None) + reply = decode(output) + if not (shared.args.chat or shared.args.cai_chat): + reply = original_question + apply_extensions(reply[len(question):], "output") - yield formatted_outputs(original_question, shared.model_name) - with eval(f"generate_with_streaming({', '.join(generate_params)})") as generator: - for output in generator: + yield formatted_outputs(reply, shared.model_name) + + # Stream the reply 1 token at a time. + # This is based on the trick of using 'stopping_criteria' to create an iterator. + elif not shared.args.flexgen: + + def generate_with_callback(callback=None, **kwargs): + kwargs['stopping_criteria'].append(Stream(callback_func=callback)) + clear_torch_cache() + with torch.no_grad(): + shared.model.generate(**kwargs) + + def generate_with_streaming(**kwargs): + return Iteratorize(generate_with_callback, kwargs, callback=None) + + yield formatted_outputs(original_question, shared.model_name) + with eval(f"generate_with_streaming({', '.join(generate_params)})") as generator: + for output in generator: + if shared.soft_prompt: + output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) + reply = decode(output) + + if not (shared.args.chat or shared.args.cai_chat): + reply = original_question + apply_extensions(reply[len(question):], "output") + yield formatted_outputs(reply, shared.model_name) + + if output[-1] == n: + break + + # Stream the output naively for FlexGen since it doesn't support 'stopping_criteria' + else: + for i in range(max_new_tokens//8+1): + clear_torch_cache() + with torch.no_grad(): + output = eval(f"shared.model.generate({', '.join(generate_params)})")[0] if shared.soft_prompt: output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) reply = decode(output) @@ -196,30 +214,14 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi reply = original_question + apply_extensions(reply[len(question):], "output") yield formatted_outputs(reply, shared.model_name) - if output[-1] == n: + if np.count_nonzero(input_ids[0] == n) < np.count_nonzero(output == n): break - # Stream the output naively for FlexGen since it doesn't support 'stopping_criteria' - else: - for i in range(max_new_tokens//8+1): - clear_torch_cache() - with torch.no_grad(): - output = eval(f"shared.model.generate({', '.join(generate_params)})")[0] - if shared.soft_prompt: - output = torch.cat((input_ids[0], output[filler_input_ids.shape[1]:])) - reply = decode(output) + input_ids = np.reshape(output, (1, output.shape[0])) + if shared.soft_prompt: + inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids) - if not (shared.args.chat or shared.args.cai_chat): - reply = original_question + apply_extensions(reply[len(question):], "output") - yield formatted_outputs(reply, shared.model_name) - - if np.count_nonzero(input_ids[0] == n) < np.count_nonzero(output == n): - break - - input_ids = np.reshape(output, (1, output.shape[0])) - if shared.soft_prompt: - inputs_embeds, filler_input_ids = generate_softprompt_input_tensors(input_ids) - - t1 = time.time() - print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output)-len(original_input_ids[0]))/(t1-t0):.2f} tokens/s, {len(output)-len(original_input_ids[0])} tokens)") - return + finally: + t1 = time.time() + print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output)-len(original_input_ids[0]))/(t1-t0):.2f} tokens/s, {len(output)-len(original_input_ids[0])} tokens)") + return From 3baf5fc700c603182456e7b4c3ac4c0f5e9748e8 Mon Sep 17 00:00:00 2001 From: Aleksey Smolenchuk Date: Sat, 11 Mar 2023 21:40:01 -0800 Subject: [PATCH 16/20] Load and save chat history in utf-8 --- modules/chat.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index a0cae949..8a221526 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -291,7 +291,7 @@ def save_history(timestamp=True): fname = f"{prefix}persistent.json" if not Path('logs').exists(): Path('logs').mkdir() - with open(Path(f'logs/{fname}'), 'w') as f: + with open(Path(f'logs/{fname}'), 'w', encoding='utf-8') as f: f.write(json.dumps({'data': shared.history['internal'], 'data_visible': shared.history['visible']}, indent=2)) return Path(f'logs/{fname}') @@ -321,7 +321,7 @@ def load_history(file, name1, name2): def load_default_history(name1, name2): if Path('logs/persistent.json').exists(): - load_history(open(Path('logs/persistent.json'), 'rb').read(), name1, name2) + load_history(open(Path('logs/persistent.json'), 'rb', encoding='utf-8').read(), name1, name2) else: shared.history['internal'] = [] shared.history['visible'] = [] @@ -355,7 +355,7 @@ def load_character(_character, name1, name2): name2 = shared.settings['name2_pygmalion'] if Path(f'logs/{shared.character}_persistent.json').exists(): - load_history(open(Path(f'logs/{shared.character}_persistent.json'), 'rb').read(), name1, name2) + load_history(open(Path(f'logs/{shared.character}_persistent.json'), 'rb', encoding='utf-8').read(), name1, name2) if shared.args.cai_chat: return name2, context, generate_chat_html(shared.history['visible'], name1, name2, shared.character) From 341e13503634a0debb684105f055e09772d16c6e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 12 Mar 2023 02:53:08 -0300 Subject: [PATCH 17/20] Various fixes in chat mode --- modules/callbacks.py | 1 + modules/chat.py | 16 ++++++---------- modules/text_generation.py | 29 +++++++++++++++-------------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/modules/callbacks.py b/modules/callbacks.py index e0d1c988..faa4a5e9 100644 --- a/modules/callbacks.py +++ b/modules/callbacks.py @@ -64,6 +64,7 @@ class Iteratorize: ret = self.mfunc(callback=_callback, **self.kwargs) except ValueError: pass + clear_torch_cache() self.q.put(self.sentinel) if self.c_callback: self.c_callback(ret) diff --git a/modules/chat.py b/modules/chat.py index 69d81e94..f40f8299 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -115,18 +115,14 @@ def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical visible_text = visible_text.replace('\n', '
') text = apply_extensions(text, "input") + if custom_generate_chat_prompt is None: + prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size) + else: + prompt = custom_generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size) + # Generate reply = '' for i in range(chat_generation_attempts): - - # The prompt needs to be generated here because, as the reply - # grows, it may become necessary to remove more old messages to - # fit into the 2048 tokens window. - if custom_generate_chat_prompt is None: - prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size-len(encode(' '+reply)[0])) - else: - prompt = custom_generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size-len(encode(' '+reply)[0])) - for reply in generate_reply(f"{prompt}{' ' if len(reply) > 0 else ''}{reply}", max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, eos_token=eos_token, stopping_string=f"\n{name1}:"): # Extracting the reply @@ -160,10 +156,10 @@ def impersonate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typ if 'pygmalion' in shared.model_name.lower(): name1 = "You" + prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size, impersonate=True) reply = '' for i in range(chat_generation_attempts): - prompt = generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size-len(encode(' '+reply)[0]), impersonate=True) for reply in generate_reply(prompt+reply, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, eos_token=eos_token, stopping_string=f"\n{name2}:"): reply, next_character_found, substring_found = extract_message_from_reply(prompt, reply, name1, name2, check, impersonate=True) if not substring_found: diff --git a/modules/text_generation.py b/modules/text_generation.py index 2460df4f..7966e126 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -92,21 +92,22 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi # These models are not part of Hugging Face, so we handle them # separately and terminate the function call earlier if shared.is_RWKV: - if shared.args.no_stream: - reply = shared.model.generate(context=question, token_count=max_new_tokens, temperature=temperature, top_p=top_p, top_k=top_k) - yield formatted_outputs(reply, shared.model_name) - else: - yield formatted_outputs(question, shared.model_name) - # RWKV has proper streaming, which is very nice. - # No need to generate 8 tokens at a time. - for reply in shared.model.generate_with_streaming(context=question, token_count=max_new_tokens, temperature=temperature, top_p=top_p, top_k=top_k): + try: + if shared.args.no_stream: + reply = shared.model.generate(context=question, token_count=max_new_tokens, temperature=temperature, top_p=top_p, top_k=top_k) yield formatted_outputs(reply, shared.model_name) - - t1 = time.time() - output = encode(reply)[0] - input_ids = encode(question) - print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output)-len(input_ids[0]))/(t1-t0):.2f} tokens/s, {len(output)-len(input_ids[0])} tokens)") - return + else: + yield formatted_outputs(question, shared.model_name) + # RWKV has proper streaming, which is very nice. + # No need to generate 8 tokens at a time. + for reply in shared.model.generate_with_streaming(context=question, token_count=max_new_tokens, temperature=temperature, top_p=top_p, top_k=top_k): + yield formatted_outputs(reply, shared.model_name) + finally: + t1 = time.time() + output = encode(reply)[0] + input_ids = encode(question) + print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output)-len(input_ids[0]))/(t1-t0):.2f} tokens/s, {len(output)-len(input_ids[0])} tokens)") + return original_question = question if not (shared.args.chat or shared.args.cai_chat): From 3f7c3d6559a51a3b95667b3ff74d048ffb722484 Mon Sep 17 00:00:00 2001 From: Aleksey Smolenchuk Date: Sat, 11 Mar 2023 22:10:57 -0800 Subject: [PATCH 18/20] No need to set encoding on binary read --- modules/chat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index 8a221526..ab5dbc2d 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -321,7 +321,7 @@ def load_history(file, name1, name2): def load_default_history(name1, name2): if Path('logs/persistent.json').exists(): - load_history(open(Path('logs/persistent.json'), 'rb', encoding='utf-8').read(), name1, name2) + load_history(open(Path('logs/persistent.json'), 'rb').read(), name1, name2) else: shared.history['internal'] = [] shared.history['visible'] = [] @@ -355,7 +355,7 @@ def load_character(_character, name1, name2): name2 = shared.settings['name2_pygmalion'] if Path(f'logs/{shared.character}_persistent.json').exists(): - load_history(open(Path(f'logs/{shared.character}_persistent.json'), 'rb', encoding='utf-8').read(), name1, name2) + load_history(open(Path(f'logs/{shared.character}_persistent.json'), 'rb').read(), name1, name2) if shared.args.cai_chat: return name2, context, generate_chat_html(shared.history['visible'], name1, name2, shared.character) From e2da6b9685a40825fa9c299d676aaae1c3d21dcc Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 12 Mar 2023 03:25:56 -0300 Subject: [PATCH 19/20] Fix You You You appearing in chat mode --- modules/chat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/chat.py b/modules/chat.py index 5bf96b1a..a6167d35 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -84,6 +84,7 @@ def extract_message_from_reply(question, reply, name1, name2, check, impersonate tmp = f"\n{asker}:" for j in range(1, len(tmp)): if reply[-j:] == tmp[:j]: + reply = reply[:-j] substring_found = True return reply, next_character_found, substring_found From ad14f0e49929d426560413c0b9de19986cbeac9e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 12 Mar 2023 03:42:29 -0300 Subject: [PATCH 20/20] Fix regenerate (provisory way) --- modules/chat.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index ae089ca5..2048e2c5 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -92,7 +92,7 @@ def extract_message_from_reply(question, reply, name1, name2, check, impersonate def stop_everything_event(): shared.stop_everything = True -def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, name1, name2, context, check, chat_prompt_size, chat_generation_attempts=1): +def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, name1, name2, context, check, chat_prompt_size, chat_generation_attempts=1, regenerate=False): shared.stop_everything = False just_started = True eos_token = '\n' if check else None @@ -121,8 +121,9 @@ def chatbot_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typical else: prompt = custom_generate_chat_prompt(text, max_new_tokens, name1, name2, context, chat_prompt_size) - # Display user input and "*is typing...*" imediately - yield shared.history['visible']+[[visible_text, '*Is typing...*']] + if not regenerate: + # Display user input and "*is typing...*" imediately + yield shared.history['visible']+[[visible_text, '*Is typing...*']] # Generate reply = '' @@ -189,7 +190,7 @@ def regenerate_wrapper(text, max_new_tokens, do_sample, temperature, top_p, typi last_visible = shared.history['visible'].pop() last_internal = shared.history['internal'].pop() - for _history in chatbot_wrapper(last_internal[0], max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, name1, name2, context, check, chat_prompt_size, chat_generation_attempts): + for _history in chatbot_wrapper(last_internal[0], max_new_tokens, do_sample, temperature, top_p, typical_p, repetition_penalty, top_k, min_length, no_repeat_ngram_size, num_beams, penalty_alpha, length_penalty, early_stopping, name1, name2, context, check, chat_prompt_size, chat_generation_attempts, regenerate=True): if shared.args.cai_chat: shared.history['visible'][-1] = [last_visible[0], _history[-1][1]] yield generate_chat_html(shared.history['visible'], name1, name2, shared.character)