mirror of
https://github.com/oobabooga/text-generation-webui.git
synced 2024-12-25 13:58:56 +01:00
3e7feb699c
* many openai updates * total reorg & cleanup. * fixups * missing import os for images * +moderations, custom_stopping_strings, more fixes * fix bugs in completion streaming * moderation fix (flagged) * updated moderation categories --------- Co-authored-by: Matthew Ashton <mashton-gitlab@zhero.org>
38 lines
869 B
Python
38 lines
869 B
Python
from extensions.openai.utils import float_list_to_base64
|
|
from modules.text_generation import encode, decode
|
|
|
|
def token_count(prompt):
|
|
tokens = encode(prompt)[0]
|
|
|
|
return {
|
|
'results': [{
|
|
'tokens': len(tokens)
|
|
}]
|
|
}
|
|
|
|
|
|
def token_encode(input, encoding_format = ''):
|
|
#if isinstance(input, list):
|
|
tokens = encode(input)[0]
|
|
|
|
return {
|
|
'results': [{
|
|
'encoding_format': encoding_format,
|
|
'tokens': float_list_to_base64(tokens) if encoding_format == "base64" else tokens,
|
|
'length': len(tokens),
|
|
}]
|
|
}
|
|
|
|
|
|
def token_decode(tokens, encoding_format):
|
|
#if isinstance(input, list):
|
|
# if encoding_format == "base64":
|
|
# tokens = base64_to_float_list(tokens)
|
|
output = decode(tokens)[0]
|
|
|
|
return {
|
|
'results': [{
|
|
'text': output
|
|
}]
|
|
}
|