Mirrors/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-12 21:37:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

convert : add t5 tokenizer tests

Browse Source
This commit is contained in:
Georgi Gerganov 2024-07-02 10:39:49 +03:00
parent 6dc9eb4040
commit 9eb5d5617d
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
27 changed files with 47 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
convert-hf-to-gguf-update.py
View File

@ -45,6 +45,7 @@ class TOKENIZER_TYPE(IntEnum):
SPM = auto()
BPE = auto()
WPM = auto()
UGM = auto()
# TODO: this string has to exercise as much pre-tokenizer functionality as possible
@ -85,6 +86,7 @@ models = [
{"name": "smaug-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct", },
{"name": "poro-chat", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Poro-34B-chat", },
{"name": "jina-v2-code", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-code", },
{"name": "t5", "tokt": TOKENIZER_TYPE.UGM, "repo": "https://huggingface.co/google-t5/t5-small", },
]
@ -106,9 +108,13 @@ def download_model(model):
os.makedirs(f"models/tokenizers/{name}", exist_ok=True)
files = ["config.json", "tokenizer.json", "tokenizer_config.json"]
if tokt == TOKENIZER_TYPE.SPM:
files.append("tokenizer.model")
if tokt == TOKENIZER_TYPE.UGM:
files.append("spiece.model")
for file in files:
save_path = f"models/tokenizers/{name}/{file}"
if os.path.isfile(save_path):
@ -131,7 +137,7 @@ for model in models:
name = model["name"]
tokt = model["tokt"]
if tokt == TOKENIZER_TYPE.SPM:
if tokt == TOKENIZER_TYPE.SPM or tokt == TOKENIZER_TYPE.UGM:
continue
# Skip if the tokenizer folder does not exist or there are other download issues previously
@ -262,6 +268,7 @@ tests = [
"\n =",
"' era",
"Hello, y'all! How are you 😁 ?我想在apple工作1314151天～",
"!!!!!!",
"3",
"33",
"333",

2
models/ggml-vocab-bert-bge.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-bert-bge.gguf.out
View File

@ -31,6 +31,7 @@
1027
1005 3690
7592 1010 1061 1005 2035 999 2129 2024 2017 100 1029 1855 100 100 6207 100 100 14677 23632 22203 1811 1995
999 999 999 999 999 999
1017
3943
21211

2
models/ggml-vocab-command-r.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-command-r.gguf.out
View File

@ -31,6 +31,7 @@
206 1857
14 4515
28339 19 1770 14 1954 8 4070 1955 1933 80503 231 5691 12081 13336 2648 29325 14315 24 26 24 27 24 28 24 5123 18372
57178 10251
26
26 26
26 26 26

2
models/ggml-vocab-deepseek-coder.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-deepseek-coder.gguf.out
View File

@ -31,6 +31,7 @@
185 405
6 2895
17535 11 320 6 435 0 1717 417 340 12394 233 210 3015 19100 608 9413 2668 16 18 16 19 16 20 16 1393 169 121 239
15330 3023
18
18 18
18 18 18

2
models/ggml-vocab-deepseek-llm.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-deepseek-llm.gguf.out
View File

@ -31,6 +31,7 @@
185 403
6 2906
17464 11 320 6 436 0 1724 418 340 33701 210 3025 19017 612 9407 2681 16 18 16 19 16 20 16 1398 68940 239
15278 3033
18
18 18
18 18 18

2
models/ggml-vocab-falcon.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-falcon.gguf.out
View File

@ -31,6 +31,7 @@
1212 40
18 4932
9856 23 291 18 436 12 1265 362 299 8196 207 204 42 50087 123 2727 20300 32022 133 234 17419 30137 28 7858 181 133 236
51520
30
3138
22287

2
models/ggml-vocab-gpt-2.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-gpt-2.gguf.out
View File

@ -31,6 +31,7 @@
198 796
6 6980
15496 11 331 6 439 0 1374 389 345 30325 223 5633 22755 239 46349 111 28839 101 18040 32432 98 43291 1485 1415 24309 25465 171 121 252
13896 3228
18
2091
20370

4
models/ggml-vocab-llama-bpe.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33
@ -104,5 +106,3 @@ __ggml_vocab_test__
🚀 (normal) 😶‍🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````""""......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL
__ggml_vocab_test__
Việt
__ggml_vocab_test__

2
models/ggml-vocab-llama-bpe.gguf.out
View File

@ -31,6 +31,7 @@
198 284
6 11639
9906 11 379 65948 0 2650 527 499 27623 223 949 37046 101067 19000 23182 102301 9263 18136 16 36827 21909
17523 3001
18
1644
8765
@ -41,4 +42,3 @@
8765 8765 1644
8765 8765 8765
198 4815 15073 66597 8004 1602 2355 79772 11187 9468 248 222 320 8416 8 27623 114 102470 9468 234 104 31643 320 36773 100166 98634 8 26602 227 11410 99 247 9468 99 247 220 18 220 1644 220 8765 220 8765 18 220 8765 1644 220 8765 8765 220 8765 8765 18 220 8765 8765 1644 220 18 13 18 220 18 497 18 220 18 1131 18 220 21549 222 98629 241 45358 233 21549 237 45358 224 21549 244 21549 115 21549 253 45358 223 21549 253 21549 95 98629 227 76460 223 949 37046 101067 19000 23182 102301 9263 18136 16 36827 21909 56560 54337 19175 102118 13373 64571 34694 3114 112203 80112 3436 106451 14196 14196 74694 3089 3089 29249 17523 3001 27708 7801 358 3077 1027 364 83 820 568 596 1070 11 364 793 499 2771 30 364 44 539 2771 358 3358 1304 433 11 364 35 499 1093 1063 15600 30 1226 6 43712 264 64966 43
101798

2
models/ggml-vocab-llama-spm.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-llama-spm.gguf.out
View File

@ -31,6 +31,7 @@
29871 13 353
525 3152
15043 29892 343 29915 497 29991 1128 526 366 29871 243 162 155 132 1577 30672 31522 30505 11548 31041 30732 29896 29941 29896 29946 29896 29945 29896 30408 30739
1738 6824 21004
29871 29941
29871 29941 29941
29871 29941 29941 29941

2
models/ggml-vocab-mpt.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-mpt.gguf.out
View File

@ -31,6 +31,7 @@
187 426
8 8685
12092 13 340 8 455 2 1359 403 368 49042 212 3736 15367 41197 13610 19934 41869 21275 1012 1047 18795 40120 20422 241
18963 4672
20
1610
20084

2
models/ggml-vocab-phi-3.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-phi-3.gguf.out
View File

@ -31,6 +31,7 @@
29871 13 353
525 3152
15043 29892 343 29915 497 29991 1128 526 366 29871 243 162 155 132 1577 30672 31522 30505 11548 31041 30732 29896 29941 29896 29946 29896 29945 29896 30408 30739
1738 6824 21004
29871 29941
29871 29941 29941
29871 29941 29941 29941

2
models/ggml-vocab-qwen2.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-qwen2.gguf.out
View File

@ -31,6 +31,7 @@
198 284
6 11385
9707 11 379 64848 0 2585 525 498 26525 223 937 104100 18493 22377 99257 16 18 16 19 16 20 16 35727 21216
17085 2928
18
18 18
18 18 18

2
models/ggml-vocab-refact.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-refact.gguf.out
View File

@ -31,6 +31,7 @@
203 280
25 34666
8279 30 533 25 464 19 4971 884 844 18458 228 1018 4982 13368 2909 9513 17827 35 37 35 38 35 39 35 11873 47838
9163 3202
37
37 37
37 37 37

2
models/ggml-vocab-starcoder.gguf.inp
View File

@ -73,6 +73,8 @@ __ggml_vocab_test__
__ggml_vocab_test__
Hello, y'all! How are you 😁 ?我想在apple工作1314151天～
__ggml_vocab_test__
!!!!!!
__ggml_vocab_test__
3
__ggml_vocab_test__
33

1
models/ggml-vocab-starcoder.gguf.out
View File

@ -31,6 +31,7 @@
222 299
44 34719
8302 49 553 44 483 38 4998 904 863 18445 247 1037 4995 13379 2924 9515 17823 54 56 54 57 54 58 54 11904 47892
9221 3226
56
56 56
56 56 56