Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix bug in multigpu setups (attempt 3)

Browse Source
This commit is contained in:
oobabooga 2023-03-06 15:58:18 -03:00
parent 09a7c36e1b
commit 20bd645f6a
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/text_generation.py
View File

@ -113,7 +113,7 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi
input_ids = encode(question, max_new_tokens) input_ids = encode(question, max_new_tokens)
cuda = "" if (shared.args.cpu or shared.args.deepspeed or shared.args.flexgen) else ".cuda()" cuda = "" if (shared.args.cpu or shared.args.deepspeed or shared.args.flexgen) else ".cuda()"
n = torch.tensor(shared.tokenizer.eos_token_id) if eos_token is None else encode(eos_token)[0][-1] n = shared.tokenizer.eos_token_id if eos_token is None else int(encode(eos_token)[0][-1])
if stopping_string is not None: if stopping_string is not None:
# The stopping_criteria code below was copied from # The stopping_criteria code below was copied from
# https://github.com/PygmalionAI/gradio-ui/blob/master/src/model.py # https://github.com/PygmalionAI/gradio-ui/blob/master/src/model.py
@ -195,7 +195,7 @@ def generate_reply(question, max_new_tokens, do_sample, temperature, top_p, typi
yield formatted_outputs(reply, shared.model_name) yield formatted_outputs(reply, shared.model_name)
if not shared.args.flexgen: if not shared.args.flexgen:
if output[-1].to("cpu") == n.to("cpu"): if output[-1] == n:
break break
input_ids = torch.reshape(output, (1, output.shape[0])) input_ids = torch.reshape(output, (1, output.shape[0]))
else: else: