Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix ExLlamaV2 context length setting (closes #5750)

Browse Source
This commit is contained in:
oobabooga 2024-03-30 21:33:16 -07:00
parent 70c58b5fc2
commit 624faa1438
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
modules/models_settings.py
View File

@ -77,9 +77,10 @@ def get_model_metadata(model):
# Transformers metadata # Transformers metadata
if hf_metadata is not None: if hf_metadata is not None:
metadata = json.loads(open(path, 'r', encoding='utf-8').read()) metadata = json.loads(open(path, 'r', encoding='utf-8').read())
if 'max_position_embeddings' in metadata: for k in ['max_position_embeddings', 'max_seq_len']:
model_settings['truncation_length'] = metadata['max_position_embeddings'] if k in metadata:
model_settings['max_seq_len'] = metadata['max_position_embeddings'] model_settings['truncation_length'] = metadata[k]
model_settings['max_seq_len'] = metadata[k]
if 'rope_theta' in metadata: if 'rope_theta' in metadata:
model_settings['rope_freq_base'] = metadata['rope_theta'] model_settings['rope_freq_base'] = metadata['rope_theta']