Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-25 17:29:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make colab functional for llama.cpp

Browse Source 
- Download only Q4_K_M for GGUF repositories by default
- Use maximum n-gpu-layers by default
This commit is contained in:
oobabooga 2023-10-22 09:06:20 -07:00
parent 994502d41b
commit 613feca23b
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Colab-TextGen-GPU.ipynb
View File

@ -115,7 +115,7 @@
" output_folder = \"\"\n",
"\n",
"# Start the web UI\n",
"cmd = f\"python server.py --share\"\n",
"cmd = f\"python server.py --share --n-gpu-layers 128\"\n",
"if output_folder != \"\":\n",
" cmd += f\" --model {output_folder}\"\n",
"cmd += f\" {command_line_flags}\"\n",

5
download-model.py
View File

@ -128,6 +128,11 @@ class ModelDownloader:
if classifications[i] in ['pytorch', 'pt']:
links.pop(i)
if has_gguf and specific_file is None:
for i in range(len(classifications) - 1, -1, -1):
if 'q4_k_m' not in links[i].lower():
links.pop(i)
is_llamacpp = has_gguf and specific_file is not None
return links, sha256, is_lora, is_llamacpp