diff --git a/INSTRUCTIONS.txt b/INSTRUCTIONS.txt deleted file mode 100644 index cb655f60..00000000 --- a/INSTRUCTIONS.txt +++ /dev/null @@ -1,19 +0,0 @@ -Thank you for downloading oobabooga/text-generation-webui. -Here is how to get it up and running: - -1. Run the "install" script to install the web UI and its requirements in this folder. -2. Run the "download-model" script to download a model of your choice. Change TextOnly variable at top of script to download only config files. -3. Run the "start-webui" script to launch the web UI. - -To add flags like --chat, --notebook, --extensions, etc, edit the -"start-webui" script using a text editor and add the desired flags -to the line that says "python server.py...". - -To get the latest updates in the future, just re-run the "install" script. -This will only install the updates, so it should be much faster. -May need to delete '\text-generation-webui\repositories\GPTQ-for-LLaMa' folder if GPTQ-for-LLaMa needs to be updated. - -You can open a command-line attached to the virtual environment by running the "micromamba-cmd" script. - -This installer uses a custom-built Windows-compatible version of bitsandbytes. Source: https://github.com/acpopescu/bitsandbytes/tree/cmake_windows -When starting the webui, you may encounter an error referencing cuda 116. Starting the webui again should allow bitsandbytes to detect the correct version. \ No newline at end of file diff --git a/download-model.bat b/download-model.bat deleted file mode 100644 index af2610c6..00000000 --- a/download-model.bat +++ /dev/null @@ -1,26 +0,0 @@ -@echo off - -SET TextOnly=False &REM True or False for Text only mode - -cd /D "%~dp0" - -set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba -set INSTALL_ENV_DIR=%cd%\installer_files\env - -if not exist "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" ( - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 -) -call "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" activate "%INSTALL_ENV_DIR%" || ( echo MicroMamba hook not found. && goto end ) - -cd text-generation-webui || goto end -goto %TextOnly% - -:False -call python download-model.py -goto end - -:True -call python download-model.py --text-only - -:end -pause diff --git a/environment_linux.sh b/environment_linux.sh new file mode 100644 index 00000000..08717db5 --- /dev/null +++ b/environment_linux.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [[ "$(pwd)" =~ " " ]]; then echo This script relies on Miniconda which can not be silently installed under a path with spaces. && exit; fi + +# config +CONDA_ROOT_PREFIX="$(pwd)/installer_files/conda" +INSTALL_ENV_DIR="$(pwd)/installer_files/env" + +# activate env +bash --init-file <(echo "source \"$CONDA_ROOT_PREFIX/etc/profile.d/conda.sh\" && conda activate \"$INSTALL_ENV_DIR\"") diff --git a/environment_macos.sh b/environment_macos.sh new file mode 100644 index 00000000..08717db5 --- /dev/null +++ b/environment_macos.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [[ "$(pwd)" =~ " " ]]; then echo This script relies on Miniconda which can not be silently installed under a path with spaces. && exit; fi + +# config +CONDA_ROOT_PREFIX="$(pwd)/installer_files/conda" +INSTALL_ENV_DIR="$(pwd)/installer_files/env" + +# activate env +bash --init-file <(echo "source \"$CONDA_ROOT_PREFIX/etc/profile.d/conda.sh\" && conda activate \"$INSTALL_ENV_DIR\"") diff --git a/environment_windows.bat b/environment_windows.bat new file mode 100644 index 00000000..e7463ebb --- /dev/null +++ b/environment_windows.bat @@ -0,0 +1,20 @@ +@echo off + +cd /D "%~dp0" + +echo "%CD%"| findstr /C:" " >nul && echo This script relies on Miniconda which can not be silently installed under a path with spaces. && goto end + +set PATH=%PATH%;%SystemRoot%\system32 + +@rem config +set CONDA_ROOT_PREFIX=%cd%\installer_files\conda +set INSTALL_ENV_DIR=%cd%\installer_files\env + +@rem activate installer env +call "%CONDA_ROOT_PREFIX%\condabin\conda.bat" activate "%INSTALL_ENV_DIR%" || ( echo. && echo Miniconda hook not found. && goto end ) + +@rem enter commands +cmd /k "%*" + +:end +pause diff --git a/install.bat b/install.bat deleted file mode 100644 index 8f904889..00000000 --- a/install.bat +++ /dev/null @@ -1,115 +0,0 @@ -@echo off - -@rem Based on the installer found here: https://github.com/Sygil-Dev/sygil-webui -@rem This script will install git and all dependencies -@rem using micromamba (an 8mb static-linked single-file binary, conda replacement). -@rem This enables a user to install this project without manually installing conda and git. - -echo WARNING: This script relies on Micromamba which may have issues on some systems when installed under a path with spaces. -echo May also have issues with long paths.&& echo. - -pause -cls - -echo What is your GPU? -echo. -echo A) NVIDIA -echo B) None (I want to run in CPU mode) -echo. -set /p "gpuchoice=Input> " -set gpuchoice=%gpuchoice:~0,1% - -if /I "%gpuchoice%" == "A" ( - set "PACKAGES_TO_INSTALL=python=3.10.9 pytorch[version=2,build=py3.10_cuda11.7*] torchvision torchaudio pytorch-cuda=11.7 cuda-toolkit ninja git" - set "CHANNEL=-c pytorch -c nvidia/label/cuda-11.7.0 -c nvidia -c conda-forge" -) else if /I "%gpuchoice%" == "B" ( - set "PACKAGES_TO_INSTALL=pytorch torchvision torchaudio cpuonly git" - set "CHANNEL=-c conda-forge -c pytorch" -) else ( - echo Invalid choice. Exiting... - exit -) - -cd /D "%~dp0" - -set PATH=%PATH%;%SystemRoot%\system32 - -set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba -set INSTALL_ENV_DIR=%cd%\installer_files\env -set MICROMAMBA_DOWNLOAD_URL=https://github.com/mamba-org/micromamba-releases/releases/download/1.4.0-0/micromamba-win-64 -set REPO_URL=https://github.com/oobabooga/text-generation-webui.git -set umamba_exists=F - -@rem figure out whether git and conda needs to be installed -call "%MAMBA_ROOT_PREFIX%\micromamba.exe" --version >nul 2>&1 -if "%ERRORLEVEL%" EQU "0" set umamba_exists=T - -@rem (if necessary) install git and conda into a contained environment -if "%PACKAGES_TO_INSTALL%" NEQ "" ( - @rem download micromamba - if "%umamba_exists%" == "F" ( - echo "Downloading Micromamba from %MICROMAMBA_DOWNLOAD_URL% to %MAMBA_ROOT_PREFIX%\micromamba.exe" - - mkdir "%MAMBA_ROOT_PREFIX%" - call curl -Lk "%MICROMAMBA_DOWNLOAD_URL%" > "%MAMBA_ROOT_PREFIX%\micromamba.exe" || ( echo. && echo Micromamba failed to download. && goto end ) - - @rem test the mamba binary - echo Micromamba version: - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" --version || ( echo. && echo Micromamba not found. && goto end ) - ) - - @rem create micromamba hook - if not exist "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" ( - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 - ) - - @rem create the installer env - if not exist "%INSTALL_ENV_DIR%" ( - echo Packages to install: %PACKAGES_TO_INSTALL% - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" create -y --prefix "%INSTALL_ENV_DIR%" %CHANNEL% %PACKAGES_TO_INSTALL% || ( echo. && echo Conda environment creation failed. && goto end ) - ) -) - -@rem check if conda environment was actually created -if not exist "%INSTALL_ENV_DIR%\python.exe" ( echo. && echo Conda environment is empty. && goto end ) - -@rem activate installer env -call "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" activate "%INSTALL_ENV_DIR%" || ( echo. && echo MicroMamba hook not found. && goto end ) - -@rem clone the repository and install the pip requirements -if exist text-generation-webui\ ( - cd text-generation-webui - git pull -) else ( - git clone https://github.com/oobabooga/text-generation-webui.git - call python -m pip install https://github.com/jllllll/bitsandbytes-windows-webui/raw/main/bitsandbytes-0.37.2-py3-none-any.whl - cd text-generation-webui || goto end -) -call python -m pip install -r requirements.txt --upgrade -call python -m pip install -r extensions\api\requirements.txt --upgrade -call python -m pip install -r extensions\elevenlabs_tts\requirements.txt --upgrade -call python -m pip install -r extensions\google_translate\requirements.txt --upgrade -call python -m pip install -r extensions\silero_tts\requirements.txt --upgrade -call python -m pip install -r extensions\whisper_stt\requirements.txt --upgrade - -@rem skip gptq install if cpu only -if /I not "%gpuchoice%" == "A" goto bandaid - -@rem download gptq and compile locally and if compile fails, install from wheel -if not exist repositories\ ( - mkdir repositories -) -cd repositories || goto end -if not exist GPTQ-for-LLaMa\ ( - git clone https://github.com/oobabooga/GPTQ-for-LLaMa.git -b cuda - cd GPTQ-for-LLaMa || goto end - call python -m pip install -r requirements.txt - call python setup_cuda.py install - if not exist "%INSTALL_ENV_DIR%\lib\site-packages\quant_cuda-0.0.0-py3.10-win-amd64.egg" ( - echo CUDA kernal compilation failed. Will try to install from wheel. - call python -m pip install https://github.com/jllllll/GPTQ-for-LLaMa-Wheels/raw/main/quant_cuda-0.0.0-cp310-cp310-win_amd64.whl || ( echo. && echo Wheel installation failed. && goto end ) - ) -) - -:end -pause diff --git a/micromamba-cmd.bat b/micromamba-cmd.bat deleted file mode 100644 index 355e7b43..00000000 --- a/micromamba-cmd.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo off - -cd /D "%~dp0" - -set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba -set INSTALL_ENV_DIR=%cd%\installer_files\env - -if not exist "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" ( - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 -) -call "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" activate "%INSTALL_ENV_DIR%" || ( echo MicroMamba hook not found. && goto end ) - -cmd /k "%*" - -:end -pause diff --git a/start-webui.bat b/start-webui.bat deleted file mode 100644 index 23b5b8c3..00000000 --- a/start-webui.bat +++ /dev/null @@ -1,19 +0,0 @@ -@echo off - -@echo Starting the web UI... - -cd /D "%~dp0" - -set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba -set INSTALL_ENV_DIR=%cd%\installer_files\env - -if not exist "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" ( - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 -) -call "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" activate "%INSTALL_ENV_DIR%" || ( echo MicroMamba hook not found. && goto end ) -cd text-generation-webui - -call python server.py --auto-devices --chat - -:end -pause diff --git a/start_linux.sh b/start_linux.sh new file mode 100644 index 00000000..8d3ecfc5 --- /dev/null +++ b/start_linux.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [[ "$(pwd)" =~ " " ]]; then echo This script relies on Miniconda which can not be silently installed under a path with spaces. && exit; fi + +OS_ARCH=$(uname -m) +case "${OS_ARCH}" in + x86_64*) OS_ARCH="x86_64";; + arm64*) OS_ARCH="aarch64";; + *) echo "Unknown system architecture: $OS_ARCH! This script runs only on x86_64 or arm64" && exit +esac + +# config +INSTALL_DIR="$(pwd)/installer_files" +CONDA_ROOT_PREFIX="$(pwd)/installer_files/conda" +INSTALL_ENV_DIR="$(pwd)/installer_files/env" +MINICONDA_DOWNLOAD_URL="https://repo.anaconda.com/miniconda/Miniconda3-py310_23.1.0-1-Linux-${OS_ARCH}.sh" +conda_exists="F" + +# figure out whether git and conda needs to be installed +if "$CONDA_ROOT_PREFIX/bin/conda" --version &>/dev/null; then conda_exists="T"; fi + +# (if necessary) install git and conda into a contained environment +# download miniconda +if [ "$conda_exists" == "F" ]; then + echo "Downloading Miniconda from $MINICONDA_DOWNLOAD_URL to $INSTALL_DIR/miniconda_installer.sh" + + mkdir -p "$INSTALL_DIR" + curl -Lk "$MINICONDA_DOWNLOAD_URL" > "$INSTALL_DIR/miniconda_installer.sh" + + chmod u+x "$INSTALL_DIR/miniconda_installer.sh" + bash "$INSTALL_DIR/miniconda_installer.sh" -b -p $CONDA_ROOT_PREFIX + + # test the conda binary + echo "Miniconda version:" + "$CONDA_ROOT_PREFIX/bin/conda" --version +fi + +# create the installer env +if [ ! -e "$INSTALL_ENV_DIR" ]; then + "$CONDA_ROOT_PREFIX/bin/conda" create -y -k --prefix "$INSTALL_ENV_DIR" python=3.10 +fi + +# check if conda environment was actually created +if [ ! -e "$INSTALL_ENV_DIR/bin/python" ]; then + echo "Conda environment is empty." + exit +fi + +# activate installer env +source "$CONDA_ROOT_PREFIX/etc/profile.d/conda.sh" # otherwise conda complains about 'shell not initialized' (needed when running in a script) +conda activate "$INSTALL_ENV_DIR" + +# setup installer env +python webui.py + +echo +echo "Done!" diff --git a/start_macos.sh b/start_macos.sh new file mode 100644 index 00000000..5686ce22 --- /dev/null +++ b/start_macos.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [[ "$(pwd)" =~ " " ]]; then echo This script relies on Miniconda which can not be silently installed under a path with spaces. && exit; fi + +# M Series or Intel +OS_ARCH=$(uname -m) +case "${OS_ARCH}" in + x86_64*) OS_ARCH="x86_64";; + arm64*) OS_ARCH="arm64";; + *) echo "Unknown system architecture: $OS_ARCH! This script runs only on x86_64 or arm64" && exit +esac + +# config +INSTALL_DIR="$(pwd)/installer_files" +CONDA_ROOT_PREFIX="$(pwd)/installer_files/conda" +INSTALL_ENV_DIR="$(pwd)/installer_files/env" +MINICONDA_DOWNLOAD_URL="https://repo.anaconda.com/miniconda/Miniconda3-py310_23.1.0-1-MacOSX-${OS_ARCH}.sh" +conda_exists="F" + +# figure out whether git and conda needs to be installed +if "$CONDA_ROOT_PREFIX/bin/conda" --version &>/dev/null; then conda_exists="T"; fi + +# (if necessary) install git and conda into a contained environment +# download miniconda +if [ "$conda_exists" == "F" ]; then + echo "Downloading Miniconda from $MINICONDA_DOWNLOAD_URL to $INSTALL_DIR/miniconda_installer.sh" + + mkdir -p "$INSTALL_DIR" + curl -Lk "$MINICONDA_DOWNLOAD_URL" > "$INSTALL_DIR/miniconda_installer.sh" + + chmod u+x "$INSTALL_DIR/miniconda_installer.sh" + bash "$INSTALL_DIR/miniconda_installer.sh" -b -p $CONDA_ROOT_PREFIX + + # test the conda binary + echo "Miniconda version:" + "$CONDA_ROOT_PREFIX/bin/conda" --version +fi + +# create the installer env +if [ ! -e "$INSTALL_ENV_DIR" ]; then + "$CONDA_ROOT_PREFIX/bin/conda" create -y -k --prefix "$INSTALL_ENV_DIR" python=3.10 +fi + +# check if conda environment was actually created +if [ ! -e "$INSTALL_ENV_DIR/bin/python" ]; then + echo "Conda environment is empty." + exit +fi + +# activate installer env +source "$CONDA_ROOT_PREFIX/etc/profile.d/conda.sh" # otherwise conda complains about 'shell not initialized' (needed when running in a script) +conda activate "$INSTALL_ENV_DIR" + +# setup installer env +python webui.py + +echo +echo "Done!" diff --git a/start_windows.bat b/start_windows.bat new file mode 100644 index 00000000..d647fd20 --- /dev/null +++ b/start_windows.bat @@ -0,0 +1,55 @@ +@echo off + +cd /D "%~dp0" + +echo "%CD%"| findstr /C:" " >nul && echo This script relies on Miniconda which can not be silently installed under a path with spaces. && goto end + +set PATH=%PATH%;%SystemRoot%\system32 + +@rem config +set INSTALL_DIR=%cd%\installer_files +set CONDA_ROOT_PREFIX=%cd%\installer_files\conda +set INSTALL_ENV_DIR=%cd%\installer_files\env +set MINICONDA_DOWNLOAD_URL=https://repo.anaconda.com/miniconda/Miniconda3-py310_23.1.0-1-Windows-x86_64.exe +set conda_exists=F + +@rem figure out whether git and conda needs to be installed +call "%CONDA_ROOT_PREFIX%\_conda.exe" --version >nul 2>&1 +if "%ERRORLEVEL%" EQU "0" set conda_exists=T + +@rem (if necessary) install git and conda into a contained environment +@rem download conda +if "%conda_exists%" == "F" ( + echo Downloading Miniconda from %MINICONDA_DOWNLOAD_URL% to %INSTALL_DIR%\miniconda_installer.exe + + mkdir "%INSTALL_DIR%" + call curl -Lk "%MINICONDA_DOWNLOAD_URL%" > "%INSTALL_DIR%\miniconda_installer.exe" || ( echo. && echo Miniconda failed to download. && goto end ) + + echo Installing Miniconda to %CONDA_ROOT_PREFIX% + start /wait "" "%INSTALL_DIR%\miniconda_installer.exe" /InstallationType=JustMe /NoShortcuts=1 /AddToPath=0 /RegisterPython=0 /NoRegistry=1 /S /D=%CONDA_ROOT_PREFIX% + + @rem test the conda binary + echo Miniconda version: + call "%CONDA_ROOT_PREFIX%\_conda.exe" --version || ( echo. && echo Miniconda not found. && goto end ) +) + +@rem create the installer env +if not exist "%INSTALL_ENV_DIR%" ( + echo Packages to install: %PACKAGES_TO_INSTALL% + call "%CONDA_ROOT_PREFIX%\_conda.exe" create --no-shortcuts -y -k --prefix "%INSTALL_ENV_DIR%" python=3.10 || ( echo. && echo Conda environment creation failed. && goto end ) +) + +@rem check if conda environment was actually created +if not exist "%INSTALL_ENV_DIR%\python.exe" ( echo. && echo Conda environment is empty. && goto end ) + +@rem activate installer env +call "%CONDA_ROOT_PREFIX%\condabin\conda.bat" activate "%INSTALL_ENV_DIR%" || ( echo. && echo Miniconda hook not found. && goto end ) + +@rem setup installer env +call python webui.py + +echo. +echo Done! + +:end +pause diff --git a/update_linux.sh b/update_linux.sh new file mode 100644 index 00000000..5966f9d9 --- /dev/null +++ b/update_linux.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [[ "$(pwd)" =~ " " ]]; then echo This script relies on Miniconda which can not be silently installed under a path with spaces. && exit; fi + +# config +CONDA_ROOT_PREFIX="$(pwd)/installer_files/conda" +INSTALL_ENV_DIR="$(pwd)/installer_files/env" + +# activate installer env +source "$CONDA_ROOT_PREFIX/etc/profile.d/conda.sh" # otherwise conda complains about 'shell not initialized' (needed when running in a script) +conda activate "$INSTALL_ENV_DIR" + +# update installer env +python webui.py --update + +echo +echo "Done!" diff --git a/update_macos.sh b/update_macos.sh new file mode 100644 index 00000000..5966f9d9 --- /dev/null +++ b/update_macos.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [[ "$(pwd)" =~ " " ]]; then echo This script relies on Miniconda which can not be silently installed under a path with spaces. && exit; fi + +# config +CONDA_ROOT_PREFIX="$(pwd)/installer_files/conda" +INSTALL_ENV_DIR="$(pwd)/installer_files/env" + +# activate installer env +source "$CONDA_ROOT_PREFIX/etc/profile.d/conda.sh" # otherwise conda complains about 'shell not initialized' (needed when running in a script) +conda activate "$INSTALL_ENV_DIR" + +# update installer env +python webui.py --update + +echo +echo "Done!" diff --git a/update_windows.bat b/update_windows.bat new file mode 100644 index 00000000..73241a1d --- /dev/null +++ b/update_windows.bat @@ -0,0 +1,23 @@ +@echo off + +cd /D "%~dp0" + +echo "%CD%"| findstr /C:" " >nul && echo This script relies on Miniconda which can not be silently installed under a path with spaces. && goto end + +set PATH=%PATH%;%SystemRoot%\system32 + +@rem config +set CONDA_ROOT_PREFIX=%cd%\installer_files\conda +set INSTALL_ENV_DIR=%cd%\installer_files\env + +@rem activate installer env +call "%CONDA_ROOT_PREFIX%\condabin\conda.bat" activate "%INSTALL_ENV_DIR%" || ( echo. && echo Miniconda hook not found. && goto end ) + +@rem update installer env +call python webui.py --update + +echo. +echo Done! + +:end +pause diff --git a/webui.py b/webui.py new file mode 100644 index 00000000..b118f80b --- /dev/null +++ b/webui.py @@ -0,0 +1,171 @@ +import argparse +import glob +import os +import shutil +import site +import subprocess +import sys + +script_dir = os.getcwd() + + +def run_cmd(cmd, capture_output=False, env=None): + # Run shell commands + return subprocess.run(cmd, shell=True, capture_output=capture_output, env=env) + + +def check_env(): + # If we have access to conda, we are probably in an environment + conda_not_exist = run_cmd("conda", capture_output=True).returncode + if conda_not_exist: + print("Conda is not installed. Exiting...") + sys.exit() + + # Ensure this is a new environment and not the base environment + if os.environ["CONDA_DEFAULT_ENV"] == "base": + print("Create an environment for this project and activate it. Exiting...") + sys.exit() + + +def install_dependencies(): + # Select your GPU or, choose to run in CPU mode + print("What is your GPU") + print() + print("A) NVIDIA") + print("B) AMD") + print("C) Apple M Series") + print("D) None (I want to run in CPU mode)") + print() + gpuchoice = input("Input> ").lower() + + # Install the version of PyTorch needed + if gpuchoice == "a": + run_cmd("conda install -y -k pytorch[version=2,build=py3.10_cuda11.7*] torchvision torchaudio pytorch-cuda=11.7 cuda-toolkit ninja git -c pytorch -c nvidia/label/cuda-11.7.0 -c nvidia") + elif gpuchoice == "b": + print("AMD GPUs are not supported. Exiting...") + sys.exit() + elif gpuchoice == "c" or gpuchoice == "d": + run_cmd("conda install -y -k pytorch torchvision torchaudio cpuonly git -c pytorch") + else: + print("Invalid choice. Exiting...") + sys.exit() + + # Clone webui to our computer + run_cmd("git clone https://github.com/oobabooga/text-generation-webui.git") + if sys.platform.startswith("win"): + # Fix a bitsandbytes compatibility issue with Windows + run_cmd("python -m pip install https://github.com/jllllll/bitsandbytes-windows-webui/raw/main/bitsandbytes-0.38.1-py3-none-any.whl") + + # Install the webui dependencies + update_dependencies() + + +def update_dependencies(): + os.chdir("text-generation-webui") + run_cmd("git pull") + + # Installs/Updates dependencies from all requirements.txt + run_cmd("python -m pip install -r requirements.txt --upgrade") + extensions = next(os.walk("extensions"))[1] + for extension in extensions: + extension_req_path = os.path.join("extensions", extension, "requirements.txt") + if os.path.exists(extension_req_path): + run_cmd("python -m pip install -r " + extension_req_path + " --upgrade") + + # The following dependencies are for CUDA, not CPU + # Check if the package cpuonly exists to determine if torch uses CUDA or not + cpuonly_exist = not run_cmd("conda list cpuonly | grep cpuonly", capture_output=True).returncode + if cpuonly_exist: + return + + # Finds the path to your dependencies + for sitedir in site.getsitepackages(): + if "site-packages" in sitedir: + site_packages_path = sitedir + break + + # This path is critical to installing the following dependencies + if site_packages_path is None: + print("Could not find the path to your Python packages. Exiting...") + sys.exit() + + # Fix a bitsandbytes compatibility issue with Linux + if sys.platform.startswith("linux"): + shutil.copy(os.path.join(site_packages_path, "bitsandbytes", "libbitsandbytes_cuda117.so"), os.path.join(site_packages_path, "bitsandbytes", "libbitsandbytes_cpu.so")) + + if not os.path.exists("repositories/"): + os.mkdir("repositories") + + # Install GPTQ-for-LLaMa which enables 4bit CUDA quantization + os.chdir("repositories") + if not os.path.exists("GPTQ-for-LLaMa/"): + run_cmd("git clone https://github.com/oobabooga/GPTQ-for-LLaMa.git -b cuda") + + # Install GPTQ-for-LLaMa dependencies + os.chdir("GPTQ-for-LLaMa") + run_cmd("git pull") + run_cmd("python -m pip install -r requirements.txt") + + # On some Linux distributions, g++ may not exist or be the wrong version to compile GPTQ-for-LLaMa + install_flag = True + if sys.platform.startswith("linux"): + gxx_output = run_cmd("g++ --version", capture_output=True) + if gxx_output.returncode != 0 or b"g++ (GCC) 12" in gxx_output.stdout: + # Install the correct version of g++ + run_cmd("conda install -y -k gxx_linux-64=11.2.0") + + # Activate the conda environment to compile GPTQ-for-LLaMa + conda_env_path = os.path.join(script_dir, "installer_files", "env") + conda_sh_path = os.path.join(script_dir, "installer_files", "conda", "etc", "profile.d", "conda.sh") + run_cmd(". " + conda_sh_path + " && conda activate " + conda_env_path + " && python setup_cuda.py install") + install_flag = False + + if install_flag: + run_cmd("python setup_cuda.py install") + install_flag = False + + # If the path does not exist, then the install failed + quant_cuda_path_regex = os.path.join(site_packages_path, "quant_cuda*/") + if not glob.glob(quant_cuda_path_regex): + print("CUDA kernel compilation failed.") + # Attempt installation via alternative, Windows-specific method + if sys.platform.startswith("win"): + print("Attempting installation with wheel.") + result = run_cmd("python -m pip install https://github.com/jllllll/GPTQ-for-LLaMa-Wheels/raw/main/quant_cuda-0.0.0-cp310-cp310-win_amd64.whl") + if result.returncode == 1: + print("Wheel installation failed.") + + +def download_model(): + os.chdir("text-generation-webui") + run_cmd("python download-model.py") + + +def run_model(): + os.chdir("text-generation-webui") + run_cmd("python server.py --auto-devices --chat --model-menu") + + +if __name__ == "__main__": + # Verifies we are in a conda environment + check_env() + + parser = argparse.ArgumentParser() + parser.add_argument('--update', action='store_true', help='Update the web UI.') + args = parser.parse_args() + + if args.update: + update_dependencies() + else: + # If webui has already been installed, skip and run + if not os.path.exists("text-generation-webui/"): + install_dependencies() + os.chdir(script_dir) + + # Check if a model has been downloaded yet + if len(glob.glob("text-generation-webui/models/*/")) == 0: + download_model() + os.chdir(script_dir) + + # Run the model with webui + run_model()