diff --git a/api-example.py b/api-example.py new file mode 100644 index 00000000..a5967a9d --- /dev/null +++ b/api-example.py @@ -0,0 +1,61 @@ +''' + +This is an example on how to use the API for oobabooga/text-generation-webui. + +Make sure to start the web UI with the following flags: + +python server.py --model MODEL --listen --no-stream + +Optionally, you can also add the --share flag to generate a public gradio URL, +allowing you to use the API remotely. + +''' + +import requests + +# Server address +server = "127.0.0.1" + +# Generation parameters +# Reference: https://huggingface.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig +params = { + 'max_new_tokens': 200, + 'do_sample': True, + 'temperature': 0.5, + 'top_p': 0.9, + 'typical_p': 1, + 'repetition_penalty': 1.05, + 'top_k': 0, + 'min_length': 0, + 'no_repeat_ngram_size': 0, + 'num_beams': 1, + 'penalty_alpha': 0, + 'length_penalty': 1, + 'early_stopping': False, +} + +# Input prompt +prompt = "What I would like to say is the following: " + +response = requests.post(f"http://{server}:7860/run/textgen", json={ + "data": [ + prompt, + params['max_new_tokens'], + params['do_sample'], + params['max_new_tokens'], + params['temperature'], + params['top_p'], + params['typical_p'], + params['repetition_penalty'], + params['top_k'], + params['min_length'], + params['no_repeat_ngram_size'], + params['num_beams'], + params['penalty_alpha'], + params['length_penalty'], + params['early_stopping'], + ] +}).json() + +reply = response["data"][0] +print(reply)