From 807a41cf87dc4c1cb22ad7d3548240e6da587401 Mon Sep 17 00:00:00 2001 From: jtang613 <> Date: Wed, 8 Mar 2023 21:02:25 -0500 Subject: [PATCH 01/24] Lets propose a name besides "Gradio" --- server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server.py b/server.py index 9f584ba3..877595cf 100644 --- a/server.py +++ b/server.py @@ -18,6 +18,8 @@ from modules.html_generator import generate_chat_html from modules.models import load_model, load_soft_prompt from modules.text_generation import generate_reply +PROJ_NAME='Oobabooga' + if (shared.args.chat or shared.args.cai_chat) and not shared.args.no_stream: print('Warning: chat mode currently becomes somewhat slower with text streaming on.\nConsider starting the web UI with the --no-stream option.\n') @@ -201,7 +203,7 @@ description = '\n\n# Text generation lab\nGenerate text using Large Language Mod suffix = '_pygmalion' if 'pygmalion' in shared.model_name.lower() else '' if shared.args.chat or shared.args.cai_chat: - with gr.Blocks(css=ui.css+ui.chat_css, analytics_enabled=False) as shared.gradio['interface']: + with gr.Blocks(css=ui.css+ui.chat_css, analytics_enabled=False, title=PROJ_NAME) as shared.gradio['interface']: if shared.args.cai_chat: shared.gradio['display'] = gr.HTML(value=generate_chat_html(shared.history['visible'], shared.settings[f'name1{suffix}'], shared.settings[f'name2{suffix}'], shared.character)) else: @@ -313,7 +315,7 @@ if shared.args.chat or shared.args.cai_chat: shared.gradio['interface'].load(reload_func, reload_inputs, [shared.gradio['display']], show_progress=True) elif shared.args.notebook: - with gr.Blocks(css=ui.css, analytics_enabled=False) as shared.gradio['interface']: + with gr.Blocks(css=ui.css, analytics_enabled=False, title=PROJ_NAME) as shared.gradio['interface']: gr.Markdown(description) with gr.Tab('Raw'): shared.gradio['textbox'] = gr.Textbox(value=default_text, lines=23) @@ -337,7 +339,7 @@ elif shared.args.notebook: shared.gradio['Stop'].click(None, None, None, cancels=gen_events) else: - with gr.Blocks(css=ui.css, analytics_enabled=False) as shared.gradio['interface']: + with gr.Blocks(css=ui.css, analytics_enabled=False, title=PROJ_NAME) as shared.gradio['interface']: gr.Markdown(description) with gr.Row(): with gr.Column(): From 4dd14dcab4778b2d4e031db9cdfa94a2e1fe13e6 Mon Sep 17 00:00:00 2001 From: Chimdumebi Nebolisa <78305519+MichealC0@users.noreply.github.com> Date: Thu, 9 Mar 2023 10:22:09 +0100 Subject: [PATCH 02/24] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9efacb7c..23d53604 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ The third line assumes that you have an NVIDIA GPU. pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/rocm5.2 ``` -* If you are running in CPU mode, replace the third command with this one: +* If you are running it in CPU mode, replace the third command with this one: ``` conda install pytorch torchvision torchaudio git -c pytorch From 038e90765bea63bcbcba4296813303c9fed6d374 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 09:44:08 -0300 Subject: [PATCH 03/24] Rename to "Text generation web UI" --- server.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server.py b/server.py index 877595cf..7d8792b7 100644 --- a/server.py +++ b/server.py @@ -18,8 +18,6 @@ from modules.html_generator import generate_chat_html from modules.models import load_model, load_soft_prompt from modules.text_generation import generate_reply -PROJ_NAME='Oobabooga' - if (shared.args.chat or shared.args.cai_chat) and not shared.args.no_stream: print('Warning: chat mode currently becomes somewhat slower with text streaming on.\nConsider starting the web UI with the --no-stream option.\n') @@ -199,11 +197,12 @@ shared.model, shared.tokenizer = load_model(shared.model_name) gen_events = [] default_preset = shared.settings['presets'][next((k for k in shared.settings['presets'] if re.match(k.lower(), shared.model_name.lower())), 'default')] default_text = shared.settings['prompts'][next((k for k in shared.settings['prompts'] if re.match(k.lower(), shared.model_name.lower())), 'default')] +title ='Text generation web UI' description = '\n\n# Text generation lab\nGenerate text using Large Language Models.\n' suffix = '_pygmalion' if 'pygmalion' in shared.model_name.lower() else '' if shared.args.chat or shared.args.cai_chat: - with gr.Blocks(css=ui.css+ui.chat_css, analytics_enabled=False, title=PROJ_NAME) as shared.gradio['interface']: + with gr.Blocks(css=ui.css+ui.chat_css, analytics_enabled=False, title=title) as shared.gradio['interface']: if shared.args.cai_chat: shared.gradio['display'] = gr.HTML(value=generate_chat_html(shared.history['visible'], shared.settings[f'name1{suffix}'], shared.settings[f'name2{suffix}'], shared.character)) else: @@ -315,7 +314,7 @@ if shared.args.chat or shared.args.cai_chat: shared.gradio['interface'].load(reload_func, reload_inputs, [shared.gradio['display']], show_progress=True) elif shared.args.notebook: - with gr.Blocks(css=ui.css, analytics_enabled=False, title=PROJ_NAME) as shared.gradio['interface']: + with gr.Blocks(css=ui.css, analytics_enabled=False, title=title) as shared.gradio['interface']: gr.Markdown(description) with gr.Tab('Raw'): shared.gradio['textbox'] = gr.Textbox(value=default_text, lines=23) @@ -339,7 +338,7 @@ elif shared.args.notebook: shared.gradio['Stop'].click(None, None, None, cancels=gen_events) else: - with gr.Blocks(css=ui.css, analytics_enabled=False, title=PROJ_NAME) as shared.gradio['interface']: + with gr.Blocks(css=ui.css, analytics_enabled=False, title=title) as shared.gradio['interface']: gr.Markdown(description) with gr.Row(): with gr.Column(): From e7adf5fe4efb586a7e9c9dd2c1a4e7faecb004ba Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 10:27:11 -0300 Subject: [PATCH 04/24] Add Contrastive Search preset #197 --- README.md | 1 - presets/Contrastive Search.txt | 3 +++ presets/Instruct Joi (Contrastive Search).txt | 5 ----- 3 files changed, 3 insertions(+), 6 deletions(-) create mode 100644 presets/Contrastive Search.txt delete mode 100644 presets/Instruct Joi (Contrastive Search).txt diff --git a/README.md b/README.md index 9efacb7c..62e8d725 100644 --- a/README.md +++ b/README.md @@ -190,5 +190,4 @@ For these two, please try commenting on an existing issue instead of creating a - NovelAI and KoboldAI presets: https://github.com/KoboldAI/KoboldAI-Client/wiki/Settings-Presets - Pygmalion preset, code for early stopping in chat mode, code for some of the sliders, --chat mode colors: https://github.com/PygmalionAI/gradio-ui/ - Verbose preset: Anonymous 4chan user. -- Instruct-Joi preset: https://huggingface.co/Rallio67/joi_12B_instruct_alpha - Gradio dropdown menu refresh button: https://github.com/AUTOMATIC1111/stable-diffusion-webui diff --git a/presets/Contrastive Search.txt b/presets/Contrastive Search.txt new file mode 100644 index 00000000..832bc9ca --- /dev/null +++ b/presets/Contrastive Search.txt @@ -0,0 +1,3 @@ +do_sample=False +penalty_alpha=0.6 +top_k=4 diff --git a/presets/Instruct Joi (Contrastive Search).txt b/presets/Instruct Joi (Contrastive Search).txt deleted file mode 100644 index e17f0a66..00000000 --- a/presets/Instruct Joi (Contrastive Search).txt +++ /dev/null @@ -1,5 +0,0 @@ -top_p=0.95 -temperature=0.5 -penalty_alpha=0.6 -top_k=4 -repetition_penalty=1.03 From 33414478bff6ccb99e4bceedae1510a13bf29936 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 11:13:03 -0300 Subject: [PATCH 05/24] Update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 62e8d725..65596321 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ For these two, please try commenting on an existing issue instead of creating a ## Credits +- Gradio dropdown menu refresh button: https://github.com/AUTOMATIC1111/stable-diffusion-webui +- Verbose preset: Anonymous 4chan user. - NovelAI and KoboldAI presets: https://github.com/KoboldAI/KoboldAI-Client/wiki/Settings-Presets - Pygmalion preset, code for early stopping in chat mode, code for some of the sliders, --chat mode colors: https://github.com/PygmalionAI/gradio-ui/ -- Verbose preset: Anonymous 4chan user. -- Gradio dropdown menu refresh button: https://github.com/AUTOMATIC1111/stable-diffusion-webui From 828a524f9a957f56c1985d71f941715727fd1db4 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 15:50:26 -0300 Subject: [PATCH 06/24] Add LLaMA 4-bit support --- modules/models.py | 22 +++++++++++++++++++++- modules/shared.py | 1 + requirements.txt | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/modules/models.py b/modules/models.py index 16ce6eb1..04235b52 100644 --- a/modules/models.py +++ b/modules/models.py @@ -1,5 +1,6 @@ import json import os +import sys import time import zipfile from pathlib import Path @@ -41,7 +42,7 @@ def load_model(model_name): shared.is_RWKV = model_name.lower().startswith('rwkv-') # Default settings - if not (shared.args.cpu or shared.args.load_in_8bit or shared.args.auto_devices or shared.args.disk or shared.args.gpu_memory is not None or shared.args.cpu_memory is not None or shared.args.deepspeed or shared.args.flexgen or shared.is_RWKV): + if not (shared.args.cpu or shared.args.load_in_8bit or shared.args.load_in_4bit or shared.args.auto_devices or shared.args.disk or shared.args.gpu_memory is not None or shared.args.cpu_memory is not None or shared.args.deepspeed or shared.args.flexgen or shared.is_RWKV): if any(size in shared.model_name.lower() for size in ('13b', '20b', '30b')): model = AutoModelForCausalLM.from_pretrained(Path(f"models/{shared.model_name}"), device_map='auto', load_in_8bit=True) else: @@ -86,6 +87,24 @@ def load_model(model_name): return model, tokenizer + # 4-bit LLaMA + elif shared.args.load_in_4bit: + sys.path.append(os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) + + from llama import load_quant + + path_to_model = Path(f'models/{model_name}') + pt_model = '' + if path_to_model.name.lower().startswith('llama-7b'): + pt_model = 'llama-7b-4bit.pt' + if path_to_model.name.lower().startswith('llama-13b'): + pt_model = 'llama-13b-4bit.pt' + if path_to_model.name.lower().startswith('llama-30b'): + pt_model = 'llama-30b-4bit.pt' + + model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) + model = model.to(torch.device('cuda:0')) + # Custom else: command = "AutoModelForCausalLM.from_pretrained" @@ -159,3 +178,4 @@ def load_soft_prompt(name): shared.soft_prompt_tensor = tensor return name + diff --git a/modules/shared.py b/modules/shared.py index b609045c..4c062fe9 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -68,6 +68,7 @@ parser.add_argument('--chat', action='store_true', help='Launch the web UI in ch parser.add_argument('--cai-chat', action='store_true', help='Launch the web UI in chat mode with a style similar to Character.AI\'s. If the file img_bot.png or img_bot.jpg exists in the same folder as server.py, this image will be used as the bot\'s profile picture. Similarly, img_me.png or img_me.jpg will be used as your profile picture.') parser.add_argument('--cpu', action='store_true', help='Use the CPU to generate text.') parser.add_argument('--load-in-8bit', action='store_true', help='Load the model with 8-bit precision.') +parser.add_argument('--load-in-4bit', action='store_true', help='Load the model with 4-bit precision. Currently only works with LLaMA.') parser.add_argument('--bf16', action='store_true', help='Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU.') parser.add_argument('--auto-devices', action='store_true', help='Automatically split the model across the available GPU(s) and CPU.') parser.add_argument('--disk', action='store_true', help='If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk.') diff --git a/requirements.txt b/requirements.txt index 47c56a45..6133f394 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ numpy rwkv==0.1.0 safetensors==0.2.8 sentencepiece -git+https://github.com/oobabooga/transformers@llama_push +git+https://github.com/zphang/transformers@llama_push From fd540b89309a138a17147955ecf8ea2049af4ca2 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 17:59:15 -0300 Subject: [PATCH 07/24] Use new LLaMA implementation (this will break stuff. I am sorry) https://github.com/oobabooga/text-generation-webui/wiki/LLaMA-model --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 47c56a45..6133f394 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ numpy rwkv==0.1.0 safetensors==0.2.8 sentencepiece -git+https://github.com/oobabooga/transformers@llama_push +git+https://github.com/zphang/transformers@llama_push From d41e3c233b4b4bccf6b0b36ff3f1db8701e52d5c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 18:02:44 -0300 Subject: [PATCH 08/24] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 65596321..9fe454c2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Text generation web UI -A gradio web UI for running Large Language Models like GPT-J 6B, OPT, GALACTICA, GPT-Neo, and Pygmalion. +A gradio web UI for running Large Language Models like GPT-J 6B, OPT, GALACTICA, LLaMA, and Pygmalion. Its goal is to become the [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) of text generation. @@ -27,6 +27,7 @@ Its goal is to become the [AUTOMATIC1111/stable-diffusion-webui](https://github. * [FlexGen offload](https://github.com/oobabooga/text-generation-webui/wiki/FlexGen). * [DeepSpeed ZeRO-3 offload](https://github.com/oobabooga/text-generation-webui/wiki/DeepSpeed). * Get responses via API, [with](https://github.com/oobabooga/text-generation-webui/blob/main/api-example-streaming.py) or [without](https://github.com/oobabooga/text-generation-webui/blob/main/api-example.py) streaming. +* [Supports the LLaMA model](https://github.com/oobabooga/text-generation-webui/wiki/LLaMA-model). * [Supports the RWKV model](https://github.com/oobabooga/text-generation-webui/wiki/RWKV-model). * Supports softprompts. * [Supports extensions](https://github.com/oobabooga/text-generation-webui/wiki/Extensions). From 2965aa1625a1186fcf36a559235881d1382f2366 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 20:48:51 -0300 Subject: [PATCH 09/24] Check if the .pt file exists --- modules/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/models.py b/modules/models.py index 04235b52..e10668cf 100644 --- a/modules/models.py +++ b/modules/models.py @@ -102,6 +102,10 @@ def load_model(model_name): if path_to_model.name.lower().startswith('llama-30b'): pt_model = 'llama-30b-4bit.pt' + if not Path(f"models/{pt_model}").exists(): + print(f"Could not find models/{pt_model}, exiting...") + exit() + model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) model = model.to(torch.device('cuda:0')) @@ -178,4 +182,3 @@ def load_soft_prompt(name): shared.soft_prompt_tensor = tensor return name - From 74102d5ee48fcf68939ff4fc3ca7e34e6623bcb7 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 20:51:22 -0300 Subject: [PATCH 10/24] Insert to the path instead of appending --- modules/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/models.py b/modules/models.py index e10668cf..0ad4c198 100644 --- a/modules/models.py +++ b/modules/models.py @@ -89,7 +89,7 @@ def load_model(model_name): # 4-bit LLaMA elif shared.args.load_in_4bit: - sys.path.append(os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) + sys.path.insert(0, os.path.abspath(Path("repositories/GPTQ-for-LLaMa"))) from llama import load_quant From eb0cb9b6df58c397bda377deefeb14a2c0b0e0f9 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 20:53:52 -0300 Subject: [PATCH 11/24] Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 65596321..de498fb6 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ Optionally, you can use the following command-line flags: | `--cai-chat` | Launch the web UI in chat mode with a style similar to Character.AI's. If the file `img_bot.png` or `img_bot.jpg` exists in the same folder as server.py, this image will be used as the bot's profile picture. Similarly, `img_me.png` or `img_me.jpg` will be used as your profile picture. | | `--cpu` | Use the CPU to generate text.| | `--load-in-8bit` | Load the model with 8-bit precision.| +| `--load-in-4bit` | Load the model with 4-bit precision. Currently only works with LLaMA. | | `--bf16` | Load the model with bfloat16 precision. Requires NVIDIA Ampere GPU. | | `--auto-devices` | Automatically split the model across the available GPU(s) and CPU.| | `--disk` | If the model is too large for your GPU(s) and CPU combined, send the remaining layers to the disk. | From 9849aac0f1284c5fa02509f1e197cc248e2c4700 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 9 Mar 2023 21:54:50 -0300 Subject: [PATCH 12/24] Don't show .pt models in the list --- modules/models.py | 3 +++ server.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/models.py b/modules/models.py index 0ad4c198..3e6cea18 100644 --- a/modules/models.py +++ b/modules/models.py @@ -105,6 +105,9 @@ def load_model(model_name): if not Path(f"models/{pt_model}").exists(): print(f"Could not find models/{pt_model}, exiting...") exit() + elif pt_model == '': + print(f"Could not find the .pt model for {model_name}, exiting...") + exit() model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) model = model.to(torch.device('cuda:0')) diff --git a/server.py b/server.py index 7d8792b7..c2977f41 100644 --- a/server.py +++ b/server.py @@ -37,7 +37,7 @@ def get_available_models(): if shared.args.flexgen: return sorted([re.sub('-np$', '', item.name) for item in list(Path('models/').glob('*')) if item.name.endswith('-np')], key=str.lower) else: - return sorted([item.name for item in list(Path('models/').glob('*')) if not item.name.endswith(('.txt', '-np'))], key=str.lower) + return sorted([item.name for item in list(Path('models/').glob('*')) if not item.name.endswith(('.txt', '-np', '.pt'))], key=str.lower) def get_available_presets(): return sorted(set(map(lambda x : '.'.join(str(x.name).split('.')[:-1]), Path('presets').glob('*.txt'))), key=str.lower) From 826e297b0ec40299318f1002f9165e7ac9c9c257 Mon Sep 17 00:00:00 2001 From: rohvani <3782201+rohvani@users.noreply.github.com> Date: Thu, 9 Mar 2023 18:31:32 -0800 Subject: [PATCH 13/24] add llama-65b-4bit support & multiple pt paths --- modules/models.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/modules/models.py b/modules/models.py index 3e6cea18..062ccb1f 100644 --- a/modules/models.py +++ b/modules/models.py @@ -97,19 +97,27 @@ def load_model(model_name): pt_model = '' if path_to_model.name.lower().startswith('llama-7b'): pt_model = 'llama-7b-4bit.pt' - if path_to_model.name.lower().startswith('llama-13b'): + elif path_to_model.name.lower().startswith('llama-13b'): pt_model = 'llama-13b-4bit.pt' - if path_to_model.name.lower().startswith('llama-30b'): + elif path_to_model.name.lower().startswith('llama-30b'): pt_model = 'llama-30b-4bit.pt' - - if not Path(f"models/{pt_model}").exists(): - print(f"Could not find models/{pt_model}, exiting...") - exit() - elif pt_model == '': + elif path_to_model.name.lower().startswith('llama-65b'): + pt_model = 'llama-65b-4bit.pt' + else: print(f"Could not find the .pt model for {model_name}, exiting...") exit() - model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) + # check root of models folder, and model path root + paths = [ f"{path_to_model}/{pt_model}", f"models/{pt_model}" ] + for path in [ Path(p) for p in paths ]: + if path.exists(): + pt_path = path + + if not pt_path: + print(f"Could not find {pt_model}, exiting...") + exit() + + model = load_quant(path_to_model, pt_path, 4) model = model.to(torch.device('cuda:0')) # Custom From 5ee376c580e8c2cf2e3b34e1822c43e6754b2649 Mon Sep 17 00:00:00 2001 From: rohvani <3782201+rohvani@users.noreply.github.com> Date: Thu, 9 Mar 2023 18:31:41 -0800 Subject: [PATCH 14/24] add LLaMA preset --- presets/LLaMA-Default.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 presets/LLaMA-Default.txt diff --git a/presets/LLaMA-Default.txt b/presets/LLaMA-Default.txt new file mode 100644 index 00000000..3df8209a --- /dev/null +++ b/presets/LLaMA-Default.txt @@ -0,0 +1,12 @@ +do_sample=False +temperature=0.7 +top_p=0 +typical_p=1 +repetition_penalty=1.15 +top_k=40 +num_beams=1 +penalty_alpha=0 +min_length=0 +length_penalty=1 +no_repeat_ngram_size=0 +early_stopping=True From ec3de0495c52a6d81495ac0553f4a7a886e4e0c8 Mon Sep 17 00:00:00 2001 From: Ber Zoidberg Date: Thu, 9 Mar 2023 19:08:09 -0800 Subject: [PATCH 15/24] download tokenizer when present --- download-model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/download-model.py b/download-model.py index 599418fc..27fbffda 100644 --- a/download-model.py +++ b/download-model.py @@ -107,9 +107,10 @@ def get_download_links_from_huggingface(model, branch): is_pytorch = re.match("pytorch_model.*\.bin", fname) is_safetensors = re.match("model.*\.safetensors", fname) + is_tokenizer = re.match("tokenizer.*\.model", fname) is_text = re.match(".*\.(txt|json)", fname) - if is_text or is_safetensors or is_pytorch: + if any((is_pytorch, is_safetensors, is_text, is_tokenizer)): if is_text: links.append(f"https://huggingface.co/{model}/resolve/{branch}/{fname}") classifications.append('text') From 249c268176114e72da3e82d7e2c652481060f44f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 10 Mar 2023 00:41:10 -0300 Subject: [PATCH 16/24] Fix the download script for long lists of files on HF --- download-model.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/download-model.py b/download-model.py index 599418fc..98b57bb0 100644 --- a/download-model.py +++ b/download-model.py @@ -5,7 +5,9 @@ Example: python download-model.py facebook/opt-1.3b ''' + import argparse +import base64 import json import multiprocessing import re @@ -93,14 +95,18 @@ facebook/opt-1.3b def get_download_links_from_huggingface(model, branch): base = "https://huggingface.co" page = f"/api/models/{model}/tree/{branch}?cursor=" + cursor = b"" links = [] classifications = [] has_pytorch = False has_safetensors = False - while page is not None: - content = requests.get(f"{base}{page}").content + while True: + content = requests.get(f"{base}{page}{cursor.decode()}").content + dict = json.loads(content) + if len(dict) == 0: + break for i in range(len(dict)): fname = dict[i]['path'] @@ -123,8 +129,9 @@ def get_download_links_from_huggingface(model, branch): has_pytorch = True classifications.append('pytorch') - #page = dict['nextUrl'] - page = None + cursor = base64.b64encode(f'{{"file_name":"{dict[-1]["path"]}"}}'.encode()) + b':50' + cursor = base64.b64encode(cursor) + cursor = cursor.replace(b'=', b'%3D') # If both pytorch and safetensors are available, download safetensors only if has_pytorch and has_safetensors: From 875847bf88c52166c4e9a0cc35f7e6c535b88d97 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 10 Mar 2023 00:45:28 -0300 Subject: [PATCH 17/24] Consider tokenizer a type of text --- download-model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download-model.py b/download-model.py index 27fbffda..bf94be7c 100644 --- a/download-model.py +++ b/download-model.py @@ -108,7 +108,7 @@ def get_download_links_from_huggingface(model, branch): is_pytorch = re.match("pytorch_model.*\.bin", fname) is_safetensors = re.match("model.*\.safetensors", fname) is_tokenizer = re.match("tokenizer.*\.model", fname) - is_text = re.match(".*\.(txt|json)", fname) + is_text = re.match(".*\.(txt|json)", fname) or is_tokenizer if any((is_pytorch, is_safetensors, is_text, is_tokenizer)): if is_text: From 2ac29137470396733e95e7efa77e091d5e8a5ef5 Mon Sep 17 00:00:00 2001 From: rohvani <3782201+rohvani@users.noreply.github.com> Date: Thu, 9 Mar 2023 20:13:23 -0800 Subject: [PATCH 18/24] fix reference issue --- modules/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/models.py b/modules/models.py index 062ccb1f..a2256b98 100644 --- a/modules/models.py +++ b/modules/models.py @@ -109,6 +109,7 @@ def load_model(model_name): # check root of models folder, and model path root paths = [ f"{path_to_model}/{pt_model}", f"models/{pt_model}" ] + pt_path = None for path in [ Path(p) for p in paths ]: if path.exists(): pt_path = path From ab470444591e425290db72db9ebc3127f5520449 Mon Sep 17 00:00:00 2001 From: deepdiffuser Date: Fri, 10 Mar 2023 04:29:09 -0800 Subject: [PATCH 19/24] add multi-gpu support for 4bit gptq LLaMA --- modules/models.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/models.py b/modules/models.py index 3e6cea18..14443c89 100644 --- a/modules/models.py +++ b/modules/models.py @@ -110,7 +110,18 @@ def load_model(model_name): exit() model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) - model = model.to(torch.device('cuda:0')) + + if shared.args.gpu_memory: + max_memory = {} + for i in range(len(shared.args.gpu_memory)): + max_memory[i] = f"{shared.args.gpu_memory[i]}GiB" + max_memory['cpu'] = f"{shared.args.cpu_memory or '99'}GiB" + + import accelerate + device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory) + model = accelerate.dispatch_model(model, device_map=device_map) + else: + model = model.to(torch.device('cuda:0')) # Custom else: From 9fbd60bf22c6a2e9cef0cade23a4933547df9114 Mon Sep 17 00:00:00 2001 From: deepdiffuser Date: Fri, 10 Mar 2023 05:30:47 -0800 Subject: [PATCH 20/24] add no_split_module_classes to prevent tensor split error --- modules/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/models.py b/modules/models.py index 14443c89..986cd73a 100644 --- a/modules/models.py +++ b/modules/models.py @@ -118,7 +118,7 @@ def load_model(model_name): max_memory['cpu'] = f"{shared.args.cpu_memory or '99'}GiB" import accelerate - device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory) + device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=["LLaMADecoderLayer"]) model = accelerate.dispatch_model(model, device_map=device_map) else: model = model.to(torch.device('cuda:0')) From e461c0b7a0769c4df3aa96505803b004a1071c2e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 10 Mar 2023 10:51:12 -0300 Subject: [PATCH 21/24] Move the import to the top --- modules/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/models.py b/modules/models.py index 986cd73a..f4c1071d 100644 --- a/modules/models.py +++ b/modules/models.py @@ -112,12 +112,13 @@ def load_model(model_name): model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) if shared.args.gpu_memory: + import accelerate + max_memory = {} for i in range(len(shared.args.gpu_memory)): max_memory[i] = f"{shared.args.gpu_memory[i]}GiB" max_memory['cpu'] = f"{shared.args.cpu_memory or '99'}GiB" - import accelerate device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=["LLaMADecoderLayer"]) model = accelerate.dispatch_model(model, device_map=device_map) else: From de7dd8b6aa3aa00ba629c9ba6ce1bc32bd213d2f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 10 Mar 2023 10:54:08 -0300 Subject: [PATCH 22/24] Add comments --- modules/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/models.py b/modules/models.py index f4c1071d..a5ec59d1 100644 --- a/modules/models.py +++ b/modules/models.py @@ -111,6 +111,7 @@ def load_model(model_name): model = load_quant(path_to_model, Path(f"models/{pt_model}"), 4) + # Multi-GPU setup if shared.args.gpu_memory: import accelerate @@ -121,6 +122,8 @@ def load_model(model_name): device_map = accelerate.infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=["LLaMADecoderLayer"]) model = accelerate.dispatch_model(model, device_map=device_map) + + # Single GPU else: model = model.to(torch.device('cuda:0')) From 706a03b2cb5bf3c0667d8c13b3a47f1a6e33cc81 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:02:25 -0300 Subject: [PATCH 23/24] Minor changes --- modules/models.py | 8 +++----- presets/LLaMA-Default.txt | 12 ------------ 2 files changed, 3 insertions(+), 17 deletions(-) delete mode 100644 presets/LLaMA-Default.txt diff --git a/modules/models.py b/modules/models.py index a2256b98..a23f1fa9 100644 --- a/modules/models.py +++ b/modules/models.py @@ -104,13 +104,11 @@ def load_model(model_name): elif path_to_model.name.lower().startswith('llama-65b'): pt_model = 'llama-65b-4bit.pt' else: - print(f"Could not find the .pt model for {model_name}, exiting...") - exit() + pt_model = f'{model_name}-4bit.pt' - # check root of models folder, and model path root - paths = [ f"{path_to_model}/{pt_model}", f"models/{pt_model}" ] + # Try to find the .pt both in models/ and in the subfolder pt_path = None - for path in [ Path(p) for p in paths ]: + for path in [Path(p) for p in [f"models/{pt_model}", f"{path_to_model}/{pt_model}"]]: if path.exists(): pt_path = path diff --git a/presets/LLaMA-Default.txt b/presets/LLaMA-Default.txt deleted file mode 100644 index 3df8209a..00000000 --- a/presets/LLaMA-Default.txt +++ /dev/null @@ -1,12 +0,0 @@ -do_sample=False -temperature=0.7 -top_p=0 -typical_p=1 -repetition_penalty=1.15 -top_k=40 -num_beams=1 -penalty_alpha=0 -min_length=0 -length_penalty=1 -no_repeat_ngram_size=0 -early_stopping=True From 026d60bd3424b5426c5ef80632aa6b71fe12d4c5 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 10 Mar 2023 14:01:02 -0300 Subject: [PATCH 24/24] Remove default preset that didn't do anything --- modules/shared.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 4c062fe9..2acb047f 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -42,7 +42,6 @@ settings = { 'default': 'NovelAI-Sphinx Moth', 'pygmalion-*': 'Pygmalion', 'RWKV-*': 'Naive', - '(rosey|chip|joi)_.*_instruct.*': 'Instruct Joi (Contrastive Search)' }, 'prompts': { 'default': 'Common sense questions and answers\n\nQuestion: \nFactual answer:',