From e6c631aea4dd4596606b0f058173de223909d372 Mon Sep 17 00:00:00 2001 From: draff Date: Fri, 10 Mar 2023 21:36:45 +0000 Subject: [PATCH 1/8] Replace --load-in-4bit with --llama-bits Replaces --load-in-4bit with a more flexible --llama-bits arg to allow for 2 and 3 bit models as well. This commit also fixes a loading issue with .pt files which are not in the root of the models folder --- README.md | 2 +- modules/models.py | 17 +++++++++-------- modules/shared.py | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c329913d..5c560172 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Optionally, you can use the following command-line flags: | `--cai-chat` | Launch the web UI in chat mode with a style similar to Character.AI's. If the file `img_bot.png` or `img_bot.jpg` exists in the same folder as server.py, this image will be used as the bot's profile picture. Similarly, `img_me.png` or `img_me.jpg` will be used as your profile picture. | | `--cpu` | Use the CPU to generate text.| | `--load-in-8bit` | Load the model with 8-bit precision.| -| `--load-in-4bit` | Load the model with 4-bit precision. Currently only works with LLaMA. | +| `--llama-bits` | Load LLaMA models with specified precision. 2, 3 and 4 bit are supported, use standard `--load-in-8bit` for 8bit precision. | | `--bf16` | Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU. | | `--auto-devices` | Automatically split the model across the available GPU(s) and CPU.| | `--disk` | If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk. | diff --git a/modules/models.py b/modules/models.py index f31d8b0d..467ffbee 100644 --- a/modules/models.py +++ b/modules/models.py @@ -42,7 +42,7 @@ def load_model(model_name): shared.is_RWKV = model_name.lower().startswith('rwkv-') # Default settings - if not (shared.args.cpu or shared.args.load_in_8bit or shared.args.load_in_4bit or shared.args.auto_devices or shared.args.disk or shared.args.gpu_memory is not None or shared.args.cpu_memory is not None or shared.args.deepspeed or shared.args.flexgen or shared.is_RWKV): + if not (shared.args.cpu or shared.args.load_in_8bit or shared.args.llama_bits>0 or shared.args.auto_devices or shared.args.disk or shared.args.gpu_memory is not None or shared.args.cpu_memory is not None or shared.args.deepspeed or shared.args.flexgen or shared.is_RWKV): if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')): model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True) else: @@ -88,23 +88,24 @@ def load_model(model_name): return model, tokenizer # 4-bit LLaMA - elif shared.args.load_in_4bit: + elif shared.args.llama_bits>0: sys.path.insert(0, os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) + bits = shared.args.llama_bits from llama import load_quant path_to_model = Path(f'models/{model_name}') pt_model = '' if path_to_model.name.lower().startswith('llama-7b'): - pt_model = 'llama-7b-4bit.pt' + pt_model = f'llama-7b-{bits}bit.pt' elif path_to_model.name.lower().startswith('llama-13b'): - pt_model = 'llama-13b-4bit.pt' + pt_model = f'llama-13b-{bits}bit.pt' elif path_to_model.name.lower().startswith('llama-30b'): - pt_model = 'llama-30b-4bit.pt' + pt_model = f'llama-30b-{bits}bit.pt' elif path_to_model.name.lower().startswith('llama-65b'): - pt_model = 'llama-65b-4bit.pt' + pt_model = f'llama-65b-{bits}bit.pt' else: - pt_model = f'{model_name}-4bit.pt' + pt_model = f'{model_name}-{bits}bit.pt' # Try to find the .pt both in models/ and in the subfolder pt_path = None @@ -116,7 +117,7 @@ def load_model(model_name): print(f"Could not find {pt_model}, exiting...") exit() - model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) + model = load_quant(path_to_model, Path(f"{pt_path}"), bits) # Multi-GPU setup if shared.args.gpu_memory: diff --git a/modules/shared.py b/modules/shared.py index 2acb047f..61d5a768 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -67,7 +67,7 @@ parser.add_argument('--chat', action='store_true', help='Launch the web UI in ch parser.add_argument('--cai-chat', action='store_true', help='Launch the web UI in chat mode with a style similar to Character.AI\'s. If the file img_bot.png or img_bot.jpg exists in the same folder as server.py, this image will be used as the bot\'s profile picture. Similarly, img_me.png or img_me.jpg will be used as your profile picture.') parser.add_argument('--cpu', action='store_true', help='Use the CPU to generate text.') parser.add_argument('--load-in-8bit', action='store_true', help='Load the model with 8-bit precision.') -parser.add_argument('--load-in-4bit', action='store_true', help='Load the model with 4-bit precision. Currently only works with LLaMA.') +parser.add_argument('--llama-bits', type=int, default=0, help='Load LLaMA models with specified precision. 2, 3 and 4 bit are supported, use standard `--load-in-8bit` for 8bit precision.') parser.add_argument('--bf16', action='store_true', help='Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU.') parser.add_argument('--auto-devices', action='store_true', help='Automatically split the model across the available GPU(s) and CPU.') parser.add_argument('--disk', action='store_true', help='If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk.') From 9ba8156a70b7d8d2cd79cac939aba22e080d8730 Mon Sep 17 00:00:00 2001 From: ItsLogic <38233332+ItsLogic@users.noreply.github.com> Date: Fri, 10 Mar 2023 22:33:58 +0000 Subject: [PATCH 2/8] remove unnecessary Path() --- modules/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/models.py b/modules/models.py index 467ffbee..3ec68f17 100644 --- a/modules/models.py +++ b/modules/models.py @@ -117,7 +117,7 @@ def load_model(model_name): print(f"Could not find {pt_model}, exiting...") exit() - model = load_quant(path_to_model, Path(f"{pt_path}"), bits) + model = load_quant(path_to_model, pt_path, bits) # Multi-GPU setup if shared.args.gpu_memory: From 804486214b5a1b07fc4c57255053593bb980d349 Mon Sep 17 00:00:00 2001 From: draff Date: Fri, 10 Mar 2023 23:21:01 +0000 Subject: [PATCH 3/8] Re-implement --load-in-4bit and update --llama-bits arg description --- README.md | 3 ++- modules/models.py | 8 ++++++-- modules/shared.py | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5c560172..76774c0b 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,8 @@ Optionally, you can use the following command-line flags: | `--cai-chat` | Launch the web UI in chat mode with a style similar to Character.AI's. If the file `img_bot.png` or `img_bot.jpg` exists in the same folder as server.py, this image will be used as the bot's profile picture. Similarly, `img_me.png` or `img_me.jpg` will be used as your profile picture. | | `--cpu` | Use the CPU to generate text.| | `--load-in-8bit` | Load the model with 8-bit precision.| -| `--llama-bits` | Load LLaMA models with specified precision. 2, 3 and 4 bit are supported, use standard `--load-in-8bit` for 8bit precision. | +| `--load-in-4bit` | Load the model with 4-bit precision. Currently only works with LLaMA.| +| `--llama-bits` | Load pre-quantized models with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA. | | `--bf16` | Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU. | | `--auto-devices` | Automatically split the model across the available GPU(s) and CPU.| | `--disk` | If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk. | diff --git a/modules/models.py b/modules/models.py index 3ec68f17..6c423a25 100644 --- a/modules/models.py +++ b/modules/models.py @@ -88,9 +88,13 @@ def load_model(model_name): return model, tokenizer # 4-bit LLaMA - elif shared.args.llama_bits>0: + elif shared.args.llama_bits>0 or shared.args.load_in_4bit: sys.path.insert(0, os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) - bits = shared.args.llama_bits + if shared.args.load_in_4bit: + bits = 4 + else: + bits = shared.args.llama_bits + from llama import load_quant diff --git a/modules/shared.py b/modules/shared.py index 61d5a768..f3f46329 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -67,7 +67,8 @@ parser.add_argument('--chat', action='store_true', help='Launch the web UI in ch parser.add_argument('--cai-chat', action='store_true', help='Launch the web UI in chat mode with a style similar to Character.AI\'s. If the file img_bot.png or img_bot.jpg exists in the same folder as server.py, this image will be used as the bot\'s profile picture. Similarly, img_me.png or img_me.jpg will be used as your profile picture.') parser.add_argument('--cpu', action='store_true', help='Use the CPU to generate text.') parser.add_argument('--load-in-8bit', action='store_true', help='Load the model with 8-bit precision.') -parser.add_argument('--llama-bits', type=int, default=0, help='Load LLaMA models with specified precision. 2, 3 and 4 bit are supported, use standard `--load-in-8bit` for 8bit precision.') +parser.add_argument('--load-in-4bit', action='store_true', help='Load the model with 4-bit precision. Currently only works with LLaMA.') +parser.add_argument('--llama-bits', type=int, default=0, help='Load pre-quantized models with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA.') parser.add_argument('--bf16', action='store_true', help='Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU.') parser.add_argument('--auto-devices', action='store_true', help='Automatically split the model across the available GPU(s) and CPU.') parser.add_argument('--disk', action='store_true', help='If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk.') From 001e638b47331f24ac967dd982f8ce4781775f7d Mon Sep 17 00:00:00 2001 From: draff Date: Fri, 10 Mar 2023 23:28:19 +0000 Subject: [PATCH 4/8] Make it actually work --- modules/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/models.py b/modules/models.py index 6c423a25..8e7caa8d 100644 --- a/modules/models.py +++ b/modules/models.py @@ -42,7 +42,7 @@ def load_model(model_name): shared.is_RWKV = model_name.lower().startswith('rwkv-') # Default settings - if not (shared.args.cpu or shared.args.load_in_8bit or shared.args.llama_bits>0 or shared.args.auto_devices or shared.args.disk or shared.args.gpu_memory is not None or shared.args.cpu_memory is not None or shared.args.deepspeed or shared.args.flexgen or shared.is_RWKV): + if not (shared.args.cpu or shared.args.load_in_8bit or shared.args.llama_bits>0 or shared.args.load_in_4bit or shared.args.auto_devices or shared.args.disk or shared.args.gpu_memory is not None or shared.args.cpu_memory is not None or shared.args.deepspeed or shared.args.flexgen or shared.is_RWKV): if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')): model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True) else: From 28fd4fc9702c9fa3a52e2ca60ca034f01cbe3be9 Mon Sep 17 00:00:00 2001 From: draff Date: Fri, 10 Mar 2023 23:34:13 +0000 Subject: [PATCH 5/8] Change wording to be consistent with other args --- README.md | 2 +- modules/shared.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 76774c0b..50d07cd6 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Optionally, you can use the following command-line flags: | `--cpu` | Use the CPU to generate text.| | `--load-in-8bit` | Load the model with 8-bit precision.| | `--load-in-4bit` | Load the model with 4-bit precision. Currently only works with LLaMA.| -| `--llama-bits` | Load pre-quantized models with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA. | +| `--llama-bits` | Load a pre-quantized model with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA. | | `--bf16` | Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU. | | `--auto-devices` | Automatically split the model across the available GPU(s) and CPU.| | `--disk` | If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk. | diff --git a/modules/shared.py b/modules/shared.py index f3f46329..3ea4ef41 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -68,7 +68,7 @@ parser.add_argument('--cai-chat', action='store_true', help='Launch the web UI i parser.add_argument('--cpu', action='store_true', help='Use the CPU to generate text.') parser.add_argument('--load-in-8bit', action='store_true', help='Load the model with 8-bit precision.') parser.add_argument('--load-in-4bit', action='store_true', help='Load the model with 4-bit precision. Currently only works with LLaMA.') -parser.add_argument('--llama-bits', type=int, default=0, help='Load pre-quantized models with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA.') +parser.add_argument('--llama-bits', type=int, default=0, help='Load a pre-quantized model with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA.') parser.add_argument('--bf16', action='store_true', help='Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU.') parser.add_argument('--auto-devices', action='store_true', help='Automatically split the model across the available GPU(s) and CPU.') parser.add_argument('--disk', action='store_true', help='If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk.') From fed3617f0727ba04d150781c698dd7df9965dc22 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 12 Mar 2023 11:12:34 -0300 Subject: [PATCH 6/8] Move LLaMA 4-bit into a separate file --- modules/models.py | 54 +++------------------------------- modules/quantized_LLaMA.py | 60 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 50 deletions(-) create mode 100644 modules/quantized_LLaMA.py diff --git a/modules/models.py b/modules/models.py index 8e7caa8d..249c05ee 100644 --- a/modules/models.py +++ b/modules/models.py @@ -42,7 +42,7 @@ def load_model(model_name): shared.is_RWKV = model_name.lower().startswith('rwkv-') # Default settings - if not (shared.args.cpu or shared.args.load_in_8bit or shared.args.llama_bits>0 or shared.args.load_in_4bit or shared.args.auto_devices or shared.args.disk or shared.args.gpu_memory is not None or shared.args.cpu_memory is not None or shared.args.deepspeed or shared.args.flexgen or shared.is_RWKV): + if not any([shared.args.cpu, shared.args.load_in_8bit, shared.args.load_in_4bit, shared.args.llama_bits > 0, shared.args.auto_devices, shared.args.disk, shared.args.gpu_memory is not None, shared.args.cpu_memory is not None, shared.args.deepspeed, shared.args.flexgen, shared.is_RWKV]): if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')): model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True) else: @@ -88,56 +88,10 @@ def load_model(model_name): return model, tokenizer # 4-bit LLaMA - elif shared.args.llama_bits>0 or shared.args.load_in_4bit: - sys.path.insert(0, os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) - if shared.args.load_in_4bit: - bits = 4 - else: - bits = shared.args.llama_bits - + elif shared.args.llama_bits > 0 or shared.args.load_in_4bit: + from modules.quantized_LLaMA import load_quantized_LLaMA - from llama import load_quant - - path_to_model = Path(f'models/{model_name}') - pt_model = '' - if path_to_model.name.lower().startswith('llama-7b'): - pt_model = f'llama-7b-{bits}bit.pt' - elif path_to_model.name.lower().startswith('llama-13b'): - pt_model = f'llama-13b-{bits}bit.pt' - elif path_to_model.name.lower().startswith('llama-30b'): - pt_model = f'llama-30b-{bits}bit.pt' - elif path_to_model.name.lower().startswith('llama-65b'): - pt_model = f'llama-65b-{bits}bit.pt' - else: - pt_model = f'{model_name}-{bits}bit.pt' - - # Try to find the .pt both in models/ and in the subfolder - pt_path = None - for path in [Path(p) for p in [f"models/{pt_model}", f"{path_to_model}/{pt_model}"]]: - if path.exists(): - pt_path = path - - if not pt_path: - print(f"Could not find {pt_model}, exiting...") - exit() - - model = load_quant(path_to_model, pt_path, bits) - - # Multi-GPU setup - if shared.args.gpu_memory: - import accelerate - - max_memory = {} - for i in range(len(shared.args.gpu_memory)): - max_memory[i] = f"{shared.args.gpu_memory[i]}GiB" - max_memory['cpu'] = f"{shared.args.cpu_memory or '99'}GiB" - - device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=["LLaMADecoderLayer"]) - model = accelerate.dispatch_model(model, device_map=device_map) - - # Single GPU - else: - model = model.to(torch.device('cuda:0')) + model = load_quantized_LLaMA(model_name) # Custom else: diff --git a/modules/quantized_LLaMA.py b/modules/quantized_LLaMA.py new file mode 100644 index 00000000..e4817da5 --- /dev/null +++ b/modules/quantized_LLaMA.py @@ -0,0 +1,60 @@ +import os +import sys +from pathlib import Path + +import accelerate +import torch + +import modules.shared as shared + +sys.path.insert(0, os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) +from llama import load_quant + + +# 4-bit LLaMA +def load_quantized_LLaMA(model_name): + if shared.args.load_in_4bit: + bits = 4 + else: + bits = shared.args.llama_bits + + path_to_model = Path(f'models/{model_name}') + pt_model = '' + if path_to_model.name.lower().startswith('llama-7b'): + pt_model = f'llama-7b-{bits}bit.pt' + elif path_to_model.name.lower().startswith('llama-13b'): + pt_model = f'llama-13b-{bits}bit.pt' + elif path_to_model.name.lower().startswith('llama-30b'): + pt_model = f'llama-30b-{bits}bit.pt' + elif path_to_model.name.lower().startswith('llama-65b'): + pt_model = f'llama-65b-{bits}bit.pt' + else: + pt_model = f'{model_name}-{bits}bit.pt' + + # Try to find the .pt both in models/ and in the subfolder + pt_path = None + for path in [Path(p) for p in [f"models/{pt_model}", f"{path_to_model}/{pt_model}"]]: + if path.exists(): + pt_path = path + + if not pt_path: + print(f"Could not find {pt_model}, exiting...") + exit() + + model = load_quant(path_to_model, pt_path, bits) + + # Multi-GPU setup + if shared.args.gpu_memory: + max_memory = {} + for i in range(len(shared.args.gpu_memory)): + max_memory[i] = f"{shared.args.gpu_memory[i]}GiB" + max_memory['cpu'] = f"{shared.args.cpu_memory or '99'}GiB" + + device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=["LLaMADecoderLayer"]) + model = accelerate.dispatch_model(model, device_map=device_map) + + # Single GPU + else: + model = model.to(torch.device('cuda:0')) + + return model From 65dda28c9dc71a2ad6bf6c627822ea2ae9d7878a Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 12 Mar 2023 11:19:07 -0300 Subject: [PATCH 7/8] Rename --llama-bits to --gptq-bits --- modules/models.py | 4 ++-- modules/quantized_LLaMA.py | 2 +- modules/shared.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/models.py b/modules/models.py index 249c05ee..7d094ed5 100644 --- a/modules/models.py +++ b/modules/models.py @@ -42,7 +42,7 @@ def load_model(model_name): shared.is_RWKV = model_name.lower().startswith('rwkv-') # Default settings - if not any([shared.args.cpu, shared.args.load_in_8bit, shared.args.load_in_4bit, shared.args.llama_bits > 0, shared.args.auto_devices, shared.args.disk, shared.args.gpu_memory is not None, shared.args.cpu_memory is not None, shared.args.deepspeed, shared.args.flexgen, shared.is_RWKV]): + if not any([shared.args.cpu, shared.args.load_in_8bit, shared.args.load_in_4bit, shared.args.gptq_bits > 0, shared.args.auto_devices, shared.args.disk, shared.args.gpu_memory is not None, shared.args.cpu_memory is not None, shared.args.deepspeed, shared.args.flexgen, shared.is_RWKV]): if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')): model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True) else: @@ -88,7 +88,7 @@ def load_model(model_name): return model, tokenizer # 4-bit LLaMA - elif shared.args.llama_bits > 0 or shared.args.load_in_4bit: + elif shared.args.gptq_bits > 0 or shared.args.load_in_4bit: from modules.quantized_LLaMA import load_quantized_LLaMA model = load_quantized_LLaMA(model_name) diff --git a/modules/quantized_LLaMA.py b/modules/quantized_LLaMA.py index e4817da5..ca4eebf2 100644 --- a/modules/quantized_LLaMA.py +++ b/modules/quantized_LLaMA.py @@ -16,7 +16,7 @@ def load_quantized_LLaMA(model_name): if shared.args.load_in_4bit: bits = 4 else: - bits = shared.args.llama_bits + bits = shared.args.gptq_bits path_to_model = Path(f'models/{model_name}') pt_model = '' diff --git a/modules/shared.py b/modules/shared.py index 3ea4ef41..627fd205 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -68,7 +68,7 @@ parser.add_argument('--cai-chat', action='store_true', help='Launch the web UI i parser.add_argument('--cpu', action='store_true', help='Use the CPU to generate text.') parser.add_argument('--load-in-8bit', action='store_true', help='Load the model with 8-bit precision.') parser.add_argument('--load-in-4bit', action='store_true', help='Load the model with 4-bit precision. Currently only works with LLaMA.') -parser.add_argument('--llama-bits', type=int, default=0, help='Load a pre-quantized model with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA.') +parser.add_argument('--gptq-bits', type=int, default=0, help='Load a pre-quantized model with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA.') parser.add_argument('--bf16', action='store_true', help='Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU.') parser.add_argument('--auto-devices', action='store_true', help='Automatically split the model across the available GPU(s) and CPU.') parser.add_argument('--disk', action='store_true', help='If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk.') From 89e9493509547e6d92c1dab724b2b0d39f389c27 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 12 Mar 2023 11:23:20 -0300 Subject: [PATCH 8/8] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50d07cd6..3d0c3a23 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ Optionally, you can use the following command-line flags: | `--cpu` | Use the CPU to generate text.| | `--load-in-8bit` | Load the model with 8-bit precision.| | `--load-in-4bit` | Load the model with 4-bit precision. Currently only works with LLaMA.| -| `--llama-bits` | Load a pre-quantized model with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA. | +| `--gptq-bits` | Load a pre-quantized model with specified precision. 2, 3, 4 and 8bit are supported. Currently only works with LLaMA. | | `--bf16` | Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU. | | `--auto-devices` | Automatically split the model across the available GPU(s) and CPU.| | `--disk` | If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk. |