Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-25 09:19:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use python module for ExLlama

Browse Source
This commit is contained in:
jllllll 2023-06-19 20:41:53 -05:00 committed by GitHub
parent 59e7ecb198
commit 8ad658ea4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/LoRA.py
View File

@ -19,11 +19,7 @@ def add_lora_to_model(lora_names):
def add_lora_exllama(lora_names): def add_lora_exllama(lora_names):
try: from exllama.lora import ExLlamaLora
from repositories.exllama.lora import ExLlamaLora
except:
logger.error("Could not find the file repositories/exllama/lora.py. Make sure that exllama is cloned inside repositories/ and is up to date.")
return
if len(lora_names) == 0: if len(lora_names) == 0:
shared.model.generator.lora = None shared.model.generator.lora = None

8
modules/exllama.py
View File

@ -3,12 +3,10 @@ from pathlib import Path
from modules import shared from modules import shared
from modules.logging_colors import logger from modules.logging_colors import logger
from modules.relative_imports import RelativeImport
with RelativeImport("repositories/exllama"): from exllama.generator import ExLlamaGenerator
from generator import ExLlamaGenerator from exllama.model import ExLlama, ExLlamaCache, ExLlamaConfig
from model import ExLlama, ExLlamaCache, ExLlamaConfig from exllama.tokenizer import ExLlamaTokenizer
from tokenizer import ExLlamaTokenizer
class ExllamaModel: class ExllamaModel:

4
requirements.txt
View File

@ -22,4 +22,6 @@ https://github.com/jllllll/bitsandbytes-windows-webui/raw/main/bitsandbytes-0.39
llama-cpp-python==0.1.62; platform_system != "Windows" llama-cpp-python==0.1.62; platform_system != "Windows"
https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.62/llama_cpp_python-0.1.62-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.62/llama_cpp_python-0.1.62-cp310-cp310-win_amd64.whl; platform_system == "Windows"
https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
https://github.com/jllllll/exllama/releases/download/0.0.1/exllama-0.0.1+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"
https://github.com/jllllll/exllama/releases/download/0.0.1/exllama-0.0.1+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"