diff --git a/README.md b/README.md index c329913d..3d0c3a23 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,8 @@ Optionally, you can use the following command-line flags: | `--cai-chat` | Launch the web UI in chat mode with a style similar to Character.AI's. If the file `img_bot.png` or `img_bot.jpg` exists in the same folder as server.py, this image will be used as the bot's profile picture. Similarly, `img_me.png` or `img_me.jpg` will be used as your profile picture. | | `--cpu` | Use the CPU to generate text.| | `--load-in-8bit` | Load the model with 8-bit precision.| -| `--load-in-4bit` | Load the model with 4-bit precision. Currently only works with LLaMA. | +| `--load-in-4bit` | Load the model with 4-bit precision. Currently only works with LLaMA.| +| `--gptq-bits` | Load a pre-quantized model with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA. | | `--bf16` | Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU. | | `--auto-devices` | Automatically split the model across the available GPU(s) and CPU.| | `--disk` | If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk. | diff --git a/modules/models.py b/modules/models.py index f31d8b0d..7d094ed5 100644 --- a/modules/models.py +++ b/modules/models.py @@ -42,7 +42,7 @@ def load_model(model_name): shared.is_RWKV = model_name.lower().startswith('rwkv-') # Default settings - if not (shared.args.cpu or shared.args.load_in_8bit or shared.args.load_in_4bit or shared.args.auto_devices or shared.args.disk or shared.args.gpu_memory is not None or shared.args.cpu_memory is not None or shared.args.deepspeed or shared.args.flexgen or shared.is_RWKV): + if not any([shared.args.cpu, shared.args.load_in_8bit, shared.args.load_in_4bit, shared.args.gptq_bits > 0, shared.args.auto_devices, shared.args.disk, shared.args.gpu_memory is not None, shared.args.cpu_memory is not None, shared.args.deepspeed, shared.args.flexgen, shared.is_RWKV]): if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')): model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True) else: @@ -88,51 +88,10 @@ def load_model(model_name): return model, tokenizer # 4-bit LLaMA - elif shared.args.load_in_4bit: - sys.path.insert(0, os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) + elif shared.args.gptq_bits > 0 or shared.args.load_in_4bit: + from modules.quantized_LLaMA import load_quantized_LLaMA - from llama import load_quant - - path_to_model = Path(f'models/{model_name}') - pt_model = '' - if path_to_model.name.lower().startswith('llama-7b'): - pt_model = 'llama-7b-4bit.pt' - elif path_to_model.name.lower().startswith('llama-13b'): - pt_model = 'llama-13b-4bit.pt' - elif path_to_model.name.lower().startswith('llama-30b'): - pt_model = 'llama-30b-4bit.pt' - elif path_to_model.name.lower().startswith('llama-65b'): - pt_model = 'llama-65b-4bit.pt' - else: - pt_model = f'{model_name}-4bit.pt' - - # Try to find the .pt both in models/ and in the subfolder - pt_path = None - for path in [Path(p) for p in [f"models/{pt_model}", f"{path_to_model}/{pt_model}"]]: - if path.exists(): - pt_path = path - - if not pt_path: - print(f"Could not find {pt_model}, exiting...") - exit() - - model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) - - # Multi-GPU setup - if shared.args.gpu_memory: - import accelerate - - max_memory = {} - for i in range(len(shared.args.gpu_memory)): - max_memory[i] = f"{shared.args.gpu_memory[i]}GiB" - max_memory['cpu'] = f"{shared.args.cpu_memory or '99'}GiB" - - device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=["LLaMADecoderLayer"]) - model = accelerate.dispatch_model(model, device_map=device_map) - - # Single GPU - else: - model = model.to(torch.device('cuda:0')) + model = load_quantized_LLaMA(model_name) # Custom else: diff --git a/modules/quantized_LLaMA.py b/modules/quantized_LLaMA.py new file mode 100644 index 00000000..ca4eebf2 --- /dev/null +++ b/modules/quantized_LLaMA.py @@ -0,0 +1,60 @@ +import os +import sys +from pathlib import Path + +import accelerate +import torch + +import modules.shared as shared + +sys.path.insert(0, os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) +from llama import load_quant + + +# 4-bit LLaMA +def load_quantized_LLaMA(model_name): + if shared.args.load_in_4bit: + bits = 4 + else: + bits = shared.args.gptq_bits + + path_to_model = Path(f'models/{model_name}') + pt_model = '' + if path_to_model.name.lower().startswith('llama-7b'): + pt_model = f'llama-7b-{bits}bit.pt' + elif path_to_model.name.lower().startswith('llama-13b'): + pt_model = f'llama-13b-{bits}bit.pt' + elif path_to_model.name.lower().startswith('llama-30b'): + pt_model = f'llama-30b-{bits}bit.pt' + elif path_to_model.name.lower().startswith('llama-65b'): + pt_model = f'llama-65b-{bits}bit.pt' + else: + pt_model = f'{model_name}-{bits}bit.pt' + + # Try to find the .pt both in models/ and in the subfolder + pt_path = None + for path in [Path(p) for p in [f"models/{pt_model}", f"{path_to_model}/{pt_model}"]]: + if path.exists(): + pt_path = path + + if not pt_path: + print(f"Could not find {pt_model}, exiting...") + exit() + + model = load_quant(path_to_model, pt_path, bits) + + # Multi-GPU setup + if shared.args.gpu_memory: + max_memory = {} + for i in range(len(shared.args.gpu_memory)): + max_memory[i] = f"{shared.args.gpu_memory[i]}GiB" + max_memory['cpu'] = f"{shared.args.cpu_memory or '99'}GiB" + + device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=["LLaMADecoderLayer"]) + model = accelerate.dispatch_model(model, device_map=device_map) + + # Single GPU + else: + model = model.to(torch.device('cuda:0')) + + return model diff --git a/modules/shared.py b/modules/shared.py index de621710..5f6c01f3 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -69,6 +69,7 @@ parser.add_argument('--cai-chat', action='store_true', help='Launch the web UI i parser.add_argument('--cpu', action='store_true', help='Use the CPU to generate text.') parser.add_argument('--load-in-8bit', action='store_true', help='Load the model with 8-bit precision.') parser.add_argument('--load-in-4bit', action='store_true', help='Load the model with 4-bit precision. Currently only works with LLaMA.') +parser.add_argument('--gptq-bits', type=int, default=0, help='Load a pre-quantized model with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA.') parser.add_argument('--bf16', action='store_true', help='Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU.') parser.add_argument('--auto-devices', action='store_true', help='Automatically split the model across the available GPU(s) and CPU.') parser.add_argument('--disk', action='store_true', help='If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk.')