From ff1cb02397002f874bde7d91256bb31a6729a577 Mon Sep 17 00:00:00 2001
From: klosax <131523366+klosax@users.noreply.github.com>
Date: Tue, 1 Aug 2023 23:17:21 +0200
Subject: [PATCH] constants.py : special tokens

---
 constants.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/constants.py b/constants.py
index d9e110b73..ae6e719fb 100644
--- a/constants.py
+++ b/constants.py
@@ -47,3 +47,8 @@ KEY_TOKENIZER_SEP_ID  = "tokenizer.ggml.seperator_token_id"
 KEY_TOKENIZER_PAD_ID  = "tokenizer.ggml.padding_token_id"
 KEY_TOKENIZER_HF_JSON = "tokenizer.huggingface.json"
 KEY_TOKENIZER_RWKV    = "tokenizer.rwkv.world"
+KEY_TOKENIZER_BOS_ID  = "tokenizer.ggml.bos_token_id"
+KEY_TOKENIZER_EOS_ID  = "tokenizer.ggml.eos_token_id"
+KEY_TOKENIZER_UNK_ID  = "tokenizer.ggml.unknown_token_id"
+KEY_TOKENIZER_SEP_ID  = "tokenizer.ggml.separator_token_id"
+KEY_TOKENIZER_PAD_ID  = "tokenizer.ggml.padding_token_id"