From ec3de0495c52a6d81495ac0553f4a7a886e4e0c8 Mon Sep 17 00:00:00 2001 From: Ber Zoidberg Date: Thu, 9 Mar 2023 19:08:09 -0800 Subject: [PATCH 1/2] download tokenizer when present --- download-model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/download-model.py b/download-model.py index 599418fc..27fbffda 100644 --- a/download-model.py +++ b/download-model.py @@ -107,9 +107,10 @@ def get_download_links_from_huggingface(model, branch): is_pytorch = re.match("pytorch_model.*\.bin", fname) is_safetensors = re.match("model.*\.safetensors", fname) + is_tokenizer = re.match("tokenizer.*\.model", fname) is_text = re.match(".*\.(txt|json)", fname) - if is_text or is_safetensors or is_pytorch: + if any((is_pytorch, is_safetensors, is_text, is_tokenizer)): if is_text: links.append(f"https://huggingface.co/{model}/resolve/{branch}/{fname}") classifications.append('text') From 875847bf88c52166c4e9a0cc35f7e6c535b88d97 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 10 Mar 2023 00:45:28 -0300 Subject: [PATCH 2/2] Consider tokenizer a type of text --- download-model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download-model.py b/download-model.py index 27fbffda..bf94be7c 100644 --- a/download-model.py +++ b/download-model.py @@ -108,7 +108,7 @@ def get_download_links_from_huggingface(model, branch): is_pytorch = re.match("pytorch_model.*\.bin", fname) is_safetensors = re.match("model.*\.safetensors", fname) is_tokenizer = re.match("tokenizer.*\.model", fname) - is_text = re.match(".*\.(txt|json)", fname) + is_text = re.match(".*\.(txt|json)", fname) or is_tokenizer if any((is_pytorch, is_safetensors, is_text, is_tokenizer)): if is_text: