Mirrors/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-12-25 22:08:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

Let model downloader download *.tiktoken as well (#4121)

Browse Source
This commit is contained in:
快乐的我531 2023-09-29 05:03:18 +08:00 committed by GitHub
parent 822ba7fcbb
commit 4e56ad55e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
download-model.py
View File

@ -91,7 +91,8 @@ class ModelDownloader:
is_safetensors = re.match(r".*\.safetensors", fname)
is_pt = re.match(r".*\.pt", fname)
is_gguf = re.match(r'.*\.gguf', fname)
is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname)
is_tiktoken = re.match(r".*\.tiktoken", fname)
is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname) or is_tiktoken
is_text = re.match(r".*\.(txt|json|py|md)", fname) or is_tokenizer
if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_tokenizer, is_text)):
if 'lfs' in dict[i]: