Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 16:17:57 +01:00
Code Issues Packages Projects Releases Wiki Activity

Minor changes

Browse Source
This commit is contained in:
oobabooga 2023-03-17 22:56:46 -03:00
parent 7994b580d5
commit e26763a510
3 changed files with 4 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -9,6 +9,8 @@ torch-dumps/*
*pycache* *pycache*
*/*pycache* */*pycache*
*/*/pycache* */*/pycache*
venv/
.venv/
settings.json settings.json
img_bot* img_bot*
@ -19,6 +21,3 @@ img_me*
!models/place-your-models-here.txt !models/place-your-models-here.txt
!softprompts/place-your-softprompts-here.txt !softprompts/place-your-softprompts-here.txt
!torch-dumps/place-your-pt-models-here.txt !torch-dumps/place-your-pt-models-here.txt
venv/
.venv/

8
modules/models.py
View File

@ -47,17 +47,13 @@ def load_model(model_name):
if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')): if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')):
model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True) model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True)
else: else:
model = AutoModelForCausalLM.from_pretrained( model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), low_cpu_mem_usage=True, torch_dtype=torch.bfloat16 if shared.args.bf16 else torch.float16)
Path(f"models/{shared.model_name}"),
low_cpu_mem_usage=True, torch_dtype=torch.bfloat16 if shared.args.bf16 else torch.float16
)
if torch.has_mps: if torch.has_mps:
device = torch.device('mps') device = torch.device('mps')
model = model.to(device) model = model.to(device)
else: else:
model = model.cuda() model = model.cuda()
# FlexGen # FlexGen
elif shared.args.flexgen: elif shared.args.flexgen:
# Initialize environment # Initialize environment
@ -106,7 +102,7 @@ def load_model(model_name):
# Custom # Custom
else: else:
params = {"low_cpu_mem_usage": True} params = {"low_cpu_mem_usage": True}
if not shared.args.cpu and not torch.cuda.is_available() and not torch.has_mps: if not any((shared.args.cpu, torch.cuda.is_available(), torch.has_mps)):
print("Warning: torch.cuda.is_available() returned False.\nThis means that no GPU has been detected.\nFalling back to CPU mode.\n") print("Warning: torch.cuda.is_available() returned False.\nThis means that no GPU has been detected.\nFalling back to CPU mode.\n")
shared.args.cpu = True shared.args.cpu = True

1
modules/text_generation.py
View File

@ -39,7 +39,6 @@ def encode(prompt, tokens_to_generate=0, add_special_tokens=True):
else: else:
return input_ids.cuda() return input_ids.cuda()
def decode(output_ids): def decode(output_ids):
# Open Assistant relies on special tokens like <|endoftext|> # Open Assistant relies on special tokens like <|endoftext|>
if re.match('(oasst|galactica)-*', shared.model_name.lower()): if re.match('(oasst|galactica)-*', shared.model_name.lower()):