Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-25 09:19:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix API example for loading models (#3101)

Browse Source
This commit is contained in:
Vadim Peretokin 2023-07-11 23:40:55 +02:00 committed by GitHub
parent f4aa11cef6
commit 987d522b55
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api-examples/api-example-model.py
View File

@ -54,7 +54,7 @@ def complex_model_load(model):
'action': 'load',
'model_name': model,
'args': {
'gptq_for_llama': False, # Use AutoGPTQ by default, set to True for gptq-for-llama
'loader': 'AutoGPTQ',
'bf16': False,
'load_in_8bit': False,
@ -74,7 +74,7 @@ def complex_model_load(model):
'rwkv_strategy': None,
'rwkv_cuda_on': False,
# b&b 4-bit
# b&b 4-bit
#'load_in_4bit': False,
#'compute_dtype': 'float16',
#'quant_type': 'nf4',
@ -148,11 +148,11 @@ if __name__ == '__main__':
except Exception as e:
print (f"❌ {model} FAIL Exception: {repr(e)}")
# 0,1,1,2,3,5,8,13, is the fibonacci sequence, the next number is 21.
# Some results below.
""" $ ./model-api-example.py
""" $ ./model-api-example.py
Model: 4bit_gpt4-x-alpaca-13b-native-4bit-128g-cuda
Lora(s): []
truncation_length = 2048