From b17893a58f1b6d11d3c39594608a421bf09928a1 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 26 Jul 2023 07:06:01 -0700 Subject: [PATCH] Revert "Add tensor split support for llama.cpp (#3171)" This reverts commit 031fe7225e91a2869647bd12f240e1ea902e30db. --- README.md | 1 - modules/llamacpp_hf.py | 7 ------- modules/llamacpp_model.py | 7 ------- modules/loaders.py | 2 -- modules/shared.py | 1 - modules/ui.py | 1 - 6 files changed, 19 deletions(-) diff --git a/README.md b/README.md index f3ab7782..073a841d 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,6 @@ Optionally, you can use the following command-line flags: | `--mlock` | Force the system to keep the model in RAM. | | `--cache-capacity CACHE_CAPACITY` | Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed. | | `--n-gpu-layers N_GPU_LAYERS` | Number of layers to offload to the GPU. Only works if llama-cpp-python was compiled with BLAS. Set this to 1000000000 to offload all layers to the GPU. | -| `--tensor_split TENSOR_SPLIT` | Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17 | | `--n_ctx N_CTX` | Size of the prompt context. | | `--llama_cpp_seed SEED` | Seed for llama-cpp models. Default 0 (random). | | `--n_gqa N_GQA` | grouped-query attention. Must be 8 for llama2 70b. | diff --git a/modules/llamacpp_hf.py b/modules/llamacpp_hf.py index 5c8c1a7a..6a4a0294 100644 --- a/modules/llamacpp_hf.py +++ b/modules/llamacpp_hf.py @@ -94,12 +94,6 @@ class LlamacppHF(PreTrainedModel): model_file = list(path.glob('*ggml*.bin'))[0] logger.info(f"llama.cpp weights detected: {model_file}\n") - - if shared.args.tensor_split is None or shared.args.tensor_split.strip() == '': - tensor_split_list = None - else: - tensor_split_list = [float(x) for x in shared.args.tensor_split.strip().split(",")] - params = { 'model_path': str(model_file), 'n_ctx': shared.args.n_ctx, @@ -110,7 +104,6 @@ class LlamacppHF(PreTrainedModel): 'use_mlock': shared.args.mlock, 'low_vram': shared.args.low_vram, 'n_gpu_layers': shared.args.n_gpu_layers, - 'tensor_split': tensor_split_list, 'rope_freq_base': 10000 * shared.args.alpha_value ** (64/63.), 'rope_freq_scale': 1.0 / shared.args.compress_pos_emb, 'n_gqa': shared.args.n_gqa or None, diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py index 69e1c160..0f9c3470 100644 --- a/modules/llamacpp_model.py +++ b/modules/llamacpp_model.py @@ -41,12 +41,6 @@ class LlamaCppModel: cache_capacity = int(shared.args.cache_capacity) logger.info("Cache capacity is " + str(cache_capacity) + " bytes") - - if shared.args.tensor_split is None or shared.args.tensor_split.strip() == '': - tensor_split_list = None - else: - tensor_split_list = [float(x) for x in shared.args.tensor_split.strip().split(",")] - params = { 'model_path': str(path), 'n_ctx': shared.args.n_ctx, @@ -57,7 +51,6 @@ class LlamaCppModel: 'use_mlock': shared.args.mlock, 'low_vram': shared.args.low_vram, 'n_gpu_layers': shared.args.n_gpu_layers, - 'tensor_split': tensor_split_list, 'rope_freq_base': 10000 * shared.args.alpha_value ** (64/63.), 'rope_freq_scale': 1.0 / shared.args.compress_pos_emb, 'n_gqa': shared.args.n_gqa or None, diff --git a/modules/loaders.py b/modules/loaders.py index acb59c65..c55cf0ff 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -33,7 +33,6 @@ loaders_and_params = { 'n_gqa', 'rms_norm_eps', 'n_gpu_layers', - 'tensor_split', 'n_batch', 'threads', 'no_mmap', @@ -48,7 +47,6 @@ loaders_and_params = { 'n_gqa', 'rms_norm_eps', 'n_gpu_layers', - 'tensor_split', 'n_batch', 'threads', 'no_mmap', diff --git a/modules/shared.py b/modules/shared.py index 978a818c..a0402878 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -125,7 +125,6 @@ parser.add_argument('--low-vram', action='store_true', help='Low VRAM Mode') parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.') parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.') parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.') -parser.add_argument('--tensor_split', type=str, default=None, help="Split the model across multiple GPUs, comma-separated list of proportions, e.g. 18,17") parser.add_argument('--n_ctx', type=int, default=2048, help='Size of the prompt context.') parser.add_argument('--llama_cpp_seed', type=int, default=0, help='Seed for llama-cpp models. Default 0 (random)') parser.add_argument('--n_gqa', type=int, default=0, help='grouped-query attention. Must be 8 for llama2 70b.') diff --git a/modules/ui.py b/modules/ui.py index 3e466ffa..d9b3a131 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -60,7 +60,6 @@ def list_model_elements(): 'low_vram', 'mlock', 'n_gpu_layers', - 'tensor_split', 'n_ctx', 'n_gqa', 'rms_norm_eps',