Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Small simplification

Browse Source
This commit is contained in:
oobabooga 2023-01-19 00:41:57 -03:00
parent 99536ef5bf
commit c90310e40e
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server.py
View File

@ -140,10 +140,10 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
preset = infile.read()
loaded_preset = inference_settings
cuda = "" if args.cpu else ".cuda()"
if not args.no_stream:
input_ids = encode(question, 1)
preset = preset.replace('max_new_tokens=tokens', 'max_new_tokens=1')
cuda = "" if args.cpu else ".cuda()"
for i in range(tokens):
output = eval(f"model.generate(input_ids, {preset}){cuda}")
reply = tokenizer.decode(output[0], skip_special_tokens=True)
@ -162,7 +162,6 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
input_ids = output
else:
input_ids = encode(question, tokens)
cuda = "" if args.cpu else ".cuda()"
if eos_token is None:
output = eval(f"model.generate(input_ids, {preset}){cuda}")
else: