Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add streaming to RWKV

Browse Source
This commit is contained in:
oobabooga 2023-02-28 00:04:04 -03:00
parent 70e522732c
commit ebd698905c
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
modules/text_generation.py
View File

@ -87,9 +87,17 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi
alpha_presence = 0.25, # Presence Penalty (as in GPT-3)
token_ban = [0], # ban the generation of some tokens
token_stop = []) # stop generation whenever you see any token here
reply = question + shared.model.generate(question, token_count=max_new_tokens, args=args, callback=None)
yield formatted_outputs(reply, None)
return formatted_outputs(reply, None)
if shared.args.no_stream:
reply = question + shared.model.generate(question, token_count=max_new_tokens, args=args, callback=None)
yield formatted_outputs(reply, None)
return formatted_outputs(reply, None)
else:
for i in range(max_new_tokens//8):
reply = question + shared.model.generate(question, token_count=8, args=args, callback=None)
yield formatted_outputs(reply, None)
question = reply
return formatted_outputs(reply, None)
original_question = question
if not (shared.args.chat or shared.args.cai_chat):