mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2024-10-29 22:20:15 +01:00
Fix per token atrributes bits (#7749)
This commit is contained in:
parent
b90dc566c1
commit
c90dbe026b
20
llama.h
20
llama.h
@ -109,16 +109,16 @@ extern "C" {
|
||||
|
||||
enum llama_token_attr {
|
||||
LLAMA_TOKEN_ATTR_UNDEFINED = 0,
|
||||
LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 1,
|
||||
LLAMA_TOKEN_ATTR_UNUSED = 1 << 2,
|
||||
LLAMA_TOKEN_ATTR_NORMAL = 1 << 3,
|
||||
LLAMA_TOKEN_ATTR_CONTROL = 1 << 4, // SPECIAL?
|
||||
LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 5,
|
||||
LLAMA_TOKEN_ATTR_BYTE = 1 << 6,
|
||||
LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 7,
|
||||
LLAMA_TOKEN_ATTR_LSTRIP = 1 << 8,
|
||||
LLAMA_TOKEN_ATTR_RSTRIP = 1 << 9,
|
||||
LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 10,
|
||||
LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,
|
||||
LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,
|
||||
LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,
|
||||
LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?
|
||||
LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
|
||||
LLAMA_TOKEN_ATTR_BYTE = 1 << 5,
|
||||
LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,
|
||||
LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,
|
||||
LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,
|
||||
LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,
|
||||
};
|
||||
|
||||
// model file types
|
||||
|
Loading…
Reference in New Issue
Block a user