Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #393 from WojtekKowaluk/mps_support

Browse Source 
Fix for MPS support on Apple Silicon
This commit is contained in:
oobabooga 2023-03-17 22:57:28 -03:00 committed by GitHub
commit bcd8afd906
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -9,6 +9,8 @@ torch-dumps/*
*pycache*
*/*pycache*
*/*/pycache*
venv/
.venv/
settings.json
img_bot*

9
modules/models.py
View File

@ -47,7 +47,12 @@ def load_model(model_name):
if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')):
model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True)
else:
model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), low_cpu_mem_usage=True, torch_dtype=torch.bfloat16 if shared.args.bf16 else torch.float16).cuda()
model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), low_cpu_mem_usage=True, torch_dtype=torch.bfloat16 if shared.args.bf16 else torch.float16)
if torch.has_mps:
device = torch.device('mps')
model = model.to(device)
else:
model = model.cuda()
# FlexGen
elif shared.args.flexgen:
@ -97,7 +102,7 @@ def load_model(model_name):
# Custom
else:
params = {"low_cpu_mem_usage": True}
if not shared.args.cpu and not torch.cuda.is_available():
if not any((shared.args.cpu, torch.cuda.is_available(), torch.has_mps)):
print("Warning: torch.cuda.is_available() returned False.\nThis means that no GPU has been detected.\nFalling back to CPU mode.\n")
shared.args.cpu = True

3
modules/text_generation.py
View File

@ -33,6 +33,9 @@ def encode(prompt, tokens_to_generate=0, add_special_tokens=True):
return input_ids.numpy()
elif shared.args.deepspeed:
return input_ids.to(device=local_rank)
elif torch.has_mps:
device = torch.device('mps')
return input_ids.to(device)
else:
return input_ids.cuda()