Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-11-22 08:07:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Specify utf-8 encoding for model metadata file open (#5125)

Browse Source
This commit is contained in:
B611 2023-12-31 05:34:32 +01:00 committed by GitHub
parent 20a2eaaf95
commit b7dd1f9542
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
modules/models_settings.py
View File

@ -35,7 +35,7 @@ def get_model_metadata(model):
path = Path(f'{shared.args.model_dir}/{model}/config.json') path = Path(f'{shared.args.model_dir}/{model}/config.json')
if path.exists(): if path.exists():
hf_metadata = json.loads(open(path, 'r').read()) hf_metadata = json.loads(open(path, 'r', encoding='utf-8').read())
else: else:
hf_metadata = None hf_metadata = None
@ -78,7 +78,7 @@ def get_model_metadata(model):
else: else:
# Transformers metadata # Transformers metadata
if hf_metadata is not None: if hf_metadata is not None:
metadata = json.loads(open(path, 'r').read()) metadata = json.loads(open(path, 'r', encoding='utf-8').read())
if 'max_position_embeddings' in metadata: if 'max_position_embeddings' in metadata:
model_settings['truncation_length'] = metadata['max_position_embeddings'] model_settings['truncation_length'] = metadata['max_position_embeddings']
model_settings['max_seq_len'] = metadata['max_position_embeddings'] model_settings['max_seq_len'] = metadata['max_position_embeddings']
@ -101,7 +101,7 @@ def get_model_metadata(model):
# Read AutoGPTQ metadata # Read AutoGPTQ metadata
path = Path(f'{shared.args.model_dir}/{model}/quantize_config.json') path = Path(f'{shared.args.model_dir}/{model}/quantize_config.json')
if path.exists(): if path.exists():
metadata = json.loads(open(path, 'r').read()) metadata = json.loads(open(path, 'r', encoding='utf-8').read())
if 'bits' in metadata: if 'bits' in metadata:
model_settings['wbits'] = metadata['bits'] model_settings['wbits'] = metadata['bits']
if 'group_size' in metadata: if 'group_size' in metadata:
@ -112,7 +112,7 @@ def get_model_metadata(model):
# Try to find the Jinja instruct template # Try to find the Jinja instruct template
path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json' path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json'
if path.exists(): if path.exists():
metadata = json.loads(open(path, 'r').read()) metadata = json.loads(open(path, 'r', encoding='utf-8').read())
if 'chat_template' in metadata: if 'chat_template' in metadata:
template = metadata['chat_template'] template = metadata['chat_template']
for k in ['eos_token', 'bos_token']: for k in ['eos_token', 'bos_token']: