Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make the Google Colab notebook functional again (attempt)

Browse Source
This commit is contained in:
oobabooga 2024-07-23 19:35:00 -07:00
parent e777b73349
commit 8b52b93e85
1 changed files with 16 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
Colab-TextGen-GPU.ipynb
View File

@ -22,7 +22,7 @@
"source": [ "source": [
"# oobabooga/text-generation-webui\n", "# oobabooga/text-generation-webui\n",
"\n", "\n",
"After running both cells, a public gradio URL will appear at the bottom in a few minutes. You can optionally generate an API link.\n", "After running both cells, a public gradio URL will appear at the bottom in around 10 minutes. You can optionally generate an API link.\n",
"\n", "\n",
"* Project page: https://github.com/oobabooga/text-generation-webui\n", "* Project page: https://github.com/oobabooga/text-generation-webui\n",
"* Gradio server status: https://status.gradio.app/" "* Gradio server status: https://status.gradio.app/"
@ -53,43 +53,27 @@
"\n", "\n",
"#@markdown If unsure about the branch, write \"main\" or leave it blank.\n", "#@markdown If unsure about the branch, write \"main\" or leave it blank.\n",
"\n", "\n",
"import torch\n", "import os\n",
"from pathlib import Path\n", "from pathlib import Path\n",
"\n", "\n",
"os.environ.pop('PYTHONPATH', None)\n",
"\n",
"if Path.cwd().name != 'text-generation-webui':\n", "if Path.cwd().name != 'text-generation-webui':\n",
" print(\"Installing the webui...\")\n", " print(\"\\033[1;32;1m\\n --> Installing the web UI. This will take a while, but after the initial setup, you can download and test as many models as you like.\\033[0;37;0m\\n\")\n",
"\n", "\n",
" !git clone https://github.com/oobabooga/text-generation-webui\n", " !git clone https://github.com/oobabooga/text-generation-webui\n",
" %cd text-generation-webui\n", " %cd text-generation-webui\n",
"\n", "\n",
" torver = torch.__version__\n", " # Install the project in an isolated environment\n",
" print(f\"TORCH: {torver}\")\n", " !GPU_CHOICE=A \\\n",
" is_cuda118 = '+cu118' in torver # 2.1.0+cu118\n", " USE_CUDA118=FALSE \\\n",
"\n", " LAUNCH_AFTER_INSTALL=FALSE \\\n",
" if is_cuda118:\n", " INSTALL_EXTENSIONS=FALSE \\\n",
" !python -m pip install --upgrade torch==2.2.1 torchvision==0.17.1 torchaudio==2.2.1 --index-url https://download.pytorch.org/whl/cu118\n", " ./start_linux.sh\n",
" else:\n",
" !python -m pip install --upgrade torch==2.2.1 torchvision==0.17.1 torchaudio==2.2.1 --index-url https://download.pytorch.org/whl/cu121\n",
"\n",
" textgen_requirements = open('requirements.txt').read().splitlines()\n",
" if is_cuda118:\n",
" textgen_requirements = [req.replace('+cu121', '+cu118').replace('+cu122', '+cu118') for req in textgen_requirements]\n",
" with open('temp_requirements.txt', 'w') as file:\n",
" file.write('\\n'.join(textgen_requirements))\n",
"\n",
" !pip install -r temp_requirements.txt --upgrade\n",
"\n",
" print(\"\\033[1;32;1m\\n --> If you see a warning about \\\"previously imported packages\\\", just ignore it.\\033[0;37;0m\")\n",
" print(\"\\033[1;32;1m\\n --> There is no need to restart the runtime.\\n\\033[0;37;0m\")\n",
"\n",
" try:\n",
" import flash_attn\n",
" except:\n",
" !pip uninstall -y flash_attn\n",
"\n", "\n",
"# Parameters\n", "# Parameters\n",
"model_url = \"https://huggingface.co/TheBloke/MythoMax-L2-13B-GPTQ\" #@param {type:\"string\"}\n", "model_url = \"https://huggingface.co/turboderp/gemma-2-9b-it-exl2\" #@param {type:\"string\"}\n",
"branch = \"gptq-4bit-32g-actorder_True\" #@param {type:\"string\"}\n", "branch = \"8.0bpw\" #@param {type:\"string\"}\n",
"command_line_flags = \"--n-gpu-layers 128 --load-in-4bit --use_double_quant\" #@param {type:\"string\"}\n", "command_line_flags = \"--n-gpu-layers 128 --load-in-4bit --use_double_quant\" #@param {type:\"string\"}\n",
"api = False #@param {type:\"boolean\"}\n", "api = False #@param {type:\"boolean\"}\n",
"\n", "\n",
@ -116,11 +100,10 @@
" output_folder = \"\"\n", " output_folder = \"\"\n",
"\n", "\n",
"# Start the web UI\n", "# Start the web UI\n",
"cmd = f\"python server.py --share\"\n", "cmd = f\"./start_linux.sh {command_line_flags} --share\"\n",
"if output_folder != \"\":\n", "if output_folder != \"\":\n",
" cmd += f\" --model {output_folder}\"\n", " cmd += f\" --model {output_folder}\"\n",
"cmd += f\" {command_line_flags}\"\n", "\n",
"print(cmd)\n",
"!$cmd" "!$cmd"
], ],
"metadata": { "metadata": {