Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-25 17:29:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

Allow loading of .safetensors through GPTQ-for-LLaMa

Browse Source
This commit is contained in:
EyeDeck 2023-03-23 21:31:34 -04:00
parent 8747c74339
commit dcfd866402
1 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/GPTQ_loader.py
View File

@ -37,21 +37,23 @@ def load_quantized(model_name):
path_to_model = Path(f'models/{model_name}') path_to_model = Path(f'models/{model_name}')
if path_to_model.name.lower().startswith('llama-7b'): if path_to_model.name.lower().startswith('llama-7b'):
pt_model = f'llama-7b-{shared.args.gptq_bits}bit.pt' pt_model = f'llama-7b-{shared.args.gptq_bits}bit'
elif path_to_model.name.lower().startswith('llama-13b'): elif path_to_model.name.lower().startswith('llama-13b'):
pt_model = f'llama-13b-{shared.args.gptq_bits}bit.pt' pt_model = f'llama-13b-{shared.args.gptq_bits}bit'
elif path_to_model.name.lower().startswith('llama-30b'): elif path_to_model.name.lower().startswith('llama-30b'):
pt_model = f'llama-30b-{shared.args.gptq_bits}bit.pt' pt_model = f'llama-30b-{shared.args.gptq_bits}bit'
elif path_to_model.name.lower().startswith('llama-65b'): elif path_to_model.name.lower().startswith('llama-65b'):
pt_model = f'llama-65b-{shared.args.gptq_bits}bit.pt' pt_model = f'llama-65b-{shared.args.gptq_bits}bit'
else: else:
pt_model = f'{model_name}-{shared.args.gptq_bits}bit.pt' pt_model = f'{model_name}-{shared.args.gptq_bits}bit'
# Try to find the .pt both in models/ and in the subfolder # Try to find the .safetensors or .pt both in models/ and in the subfolder
pt_path = None pt_path = None
for path in [Path(p) for p in [f"models/{pt_model}", f"{path_to_model}/{pt_model}"]]: for path in [Path(p+ext) for ext in ['.safetensors', '.pt'] for p in [f"models/{pt_model}", f"{path_to_model}/{pt_model}"]]:
if path.exists(): if path.exists():
print(f"Found {path}")
pt_path = path pt_path = path
break
if not pt_path: if not pt_path:
print(f"Could not find {pt_model}, exiting...") print(f"Could not find {pt_model}, exiting...")