Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-12-24 13:28:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

Remove obsolete warning

Browse Source
This commit is contained in:
oobabooga 2023-09-18 12:25:10 -07:00
parent 745807dc03
commit f0ef971edb
1 changed files with 0 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/models.py
View File

@ -111,22 +111,6 @@ def load_tokenizer(model_name, model):
use_fast=True
)
if tokenizer.__class__.__name__ == 'LlamaTokenizer':
pairs = [
['tokenizer_config.json', '516c6167c884793a738c440e29ccb80c15e1493ffc965affc69a1a8ddef4572a'],
['special_tokens_map.json', 'ff3b4a612c4e447acb02d40071bddd989fe0da87eb5b7fe0dbadfc4f74de7531']
]
for pair in pairs:
p = path_to_model / pair[0]
if p.exists():
with open(p, "rb") as f:
bytes = f.read()
file_hash = hashlib.sha256(bytes).hexdigest()
if file_hash != pair[1]:
logger.warning(f"{p} is different from the original LlamaTokenizer file. It is either customized or outdated.")
return tokenizer