From 0a0d2895374b77086e4405c42a89e2096f949772 Mon Sep 17 00:00:00 2001 From: 81300 <105078168+81300@users.noreply.github.com> Date: Wed, 1 Feb 2023 19:02:07 +0200 Subject: [PATCH] Fix issue with generating on multiple GPUs --- server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 9a430c1b..844d5e66 100644 --- a/server.py +++ b/server.py @@ -314,7 +314,7 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok t0 = time.time() if args.deepspeed: with torch.no_grad(): - output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})") + output = eval(f"model.generate(input_ids, synced_gpus=True, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})") else: output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset}){cuda}") reply = decode(output[0]) @@ -331,7 +331,7 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok for i in tqdm(range(tokens//8+1)): if args.deepspeed: with torch.no_grad(): - output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})") + output = eval(f"model.generate(input_ids, synced_gpus=True, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})") else: output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset}){cuda}") reply = decode(output[0])