Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-25 17:29:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

Autodetect llamacpp_HF loader when tokenizer exists

Browse Source
This commit is contained in:
oobabooga 2024-02-16 09:29:26 -08:00
parent 76d28eaa9e
commit 4039999be5
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/models.py
View File

@ -257,7 +257,7 @@ def llamacpp_HF_loader(model_name):
path = Path(f'{shared.args.model_dir}/{model_name}') path = Path(f'{shared.args.model_dir}/{model_name}')
# Check if a HF tokenizer is available for the model # Check if a HF tokenizer is available for the model
if all((path / file).exists() for file in ['tokenizer.json', 'tokenizer_config.json']): if all((path / file).exists() for file in ['tokenizer_config.json']):
logger.info(f'Using tokenizer from: \"{path}\"') logger.info(f'Using tokenizer from: \"{path}\"')
else: else:
logger.error("Could not load the model because a tokenizer in Transformers format was not found.") logger.error("Could not load the model because a tokenizer in Transformers format was not found.")

4
modules/models_settings.py
View File

@ -153,6 +153,8 @@ def infer_loader(model_name, model_settings):
loader = 'ExLlamav2_HF' loader = 'ExLlamav2_HF'
elif (path_to_model / 'quant_config.json').exists() or re.match(r'.*-awq', model_name.lower()): elif (path_to_model / 'quant_config.json').exists() or re.match(r'.*-awq', model_name.lower()):
loader = 'AutoAWQ' loader = 'AutoAWQ'
elif len(list(path_to_model.glob('*.gguf'))) > 0 and path_to_model.is_dir() and (path_to_model / 'tokenizer_config.json').exists():
loader = 'llamacpp_HF'
elif len(list(path_to_model.glob('*.gguf'))) > 0: elif len(list(path_to_model.glob('*.gguf'))) > 0:
loader = 'llama.cpp' loader = 'llama.cpp'
elif re.match(r'.*\.gguf', model_name.lower()): elif re.match(r'.*\.gguf', model_name.lower()):
@ -225,7 +227,7 @@ def apply_model_settings_to_state(model, state):
loader = model_settings.pop('loader') loader = model_settings.pop('loader')
# If the user is using an alternative loader for the same model type, let them keep using it # If the user is using an alternative loader for the same model type, let them keep using it
if not (loader == 'ExLlamav2_HF' and state['loader'] in ['GPTQ-for-LLaMa', 'ExLlamav2', 'AutoGPTQ']) and not (loader == 'llama.cpp' and state['loader'] in ['llamacpp_HF', 'ctransformers']): if not (loader == 'ExLlamav2_HF' and state['loader'] in ['GPTQ-for-LLaMa', 'ExLlamav2', 'AutoGPTQ']) and not (loader == 'llama.cpp' and state['loader'] in ['ctransformers']):
state['loader'] = loader state['loader'] = loader
for k in model_settings: for k in model_settings: