From 4e56ad55e1223b97616fc60e4920d796106984f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BF=AB=E4=B9=90=E7=9A=84=E6=88=91531?=
 <2302004040@qq.com>
Date: Fri, 29 Sep 2023 05:03:18 +0800
Subject: [PATCH] Let model downloader download *.tiktoken as well (#4121)

---
 download-model.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/download-model.py b/download-model.py
index d9b21d3a..d37ae32c 100644
--- a/download-model.py
+++ b/download-model.py
@@ -91,7 +91,8 @@ class ModelDownloader:
                 is_safetensors = re.match(r".*\.safetensors", fname)
                 is_pt = re.match(r".*\.pt", fname)
                 is_gguf = re.match(r'.*\.gguf', fname)
-                is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname)
+                is_tiktoken = re.match(r".*\.tiktoken", fname)
+                is_tokenizer = re.match(r"(tokenizer|ice|spiece).*\.model", fname) or is_tiktoken
                 is_text = re.match(r".*\.(txt|json|py|md)", fname) or is_tokenizer
                 if any((is_pytorch, is_safetensors, is_pt, is_gguf, is_tokenizer, is_text)):
                     if 'lfs' in dict[i]: