Mirrors/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-30 22:03:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix server.cpp code style according to review

Browse Source
This commit is contained in:
kir-gadjello 2023-11-22 22:35:57 -03:00 committed by GitHub
parent 9ad4d273e1
commit e1516709f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 263 additions and 278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
examples/server/server.cpp
View File

@ -61,14 +61,14 @@ static bool server_verbose = false;
#define LOG_WARNING(MSG, ...) server_log("WARNING", __func__, __LINE__, MSG, __VA_ARGS__) #define LOG_WARNING(MSG, ...) server_log("WARNING", __func__, __LINE__, MSG, __VA_ARGS__)
#define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__) #define LOG_INFO( MSG, ...) server_log("INFO", __func__, __LINE__, MSG, __VA_ARGS__)
json oaicompat_completion_params_parse(const json &body);
std::string format_chatml(std::vector<json> messages);
// //
// base64 utils (TODO: move to common in the future) // base64 utils (TODO: move to common in the future)
// //
json oaicompat_completion_params_parse(
const json &body);
std::string format_chatml(std::vector<json> messages);
static const std::string base64_chars = static const std::string base64_chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ" "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz" "abcdefghijklmnopqrstuvwxyz"
@ -688,8 +688,7 @@ struct llama_server_context
if (data.count("__oaicompat") != 0) { if (data.count("__oaicompat") != 0) {
slot->oaicompat = true; slot->oaicompat = true;
slot->oaicompat_model = slot->oaicompat_model = json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL));
json_value(data, "model", std::string(DEFAULT_OAICOMPAT_MODEL));
} else { } else {
slot->oaicompat = false; slot->oaicompat = false;
slot->oaicompat_model = ""; slot->oaicompat_model = "";
@ -2209,9 +2208,9 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
} }
static std::string random_string() { static std::string random_string()
std::string str( {
"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); std::string str("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
std::random_device rd; std::random_device rd;
std::mt19937 generator(rd()); std::mt19937 generator(rd());
@ -2221,17 +2220,17 @@ static std::string random_string() {
return str.substr(0, 32); // assumes 32 < number of characters in str return str.substr(0, 32); // assumes 32 < number of characters in str
} }
static std::string gen_chatcmplid() { static std::string gen_chatcmplid()
{
std::stringstream chatcmplid; std::stringstream chatcmplid;
chatcmplid << "chatcmpl-" << random_string(); chatcmplid << "chatcmpl-" << random_string();
return chatcmplid.str(); return chatcmplid.str();
} }
std::string format_chatml(std::vector<json> messages) { std::string format_chatml(std::vector<json> messages)
{
std::ostringstream chatml_msgs; std::ostringstream chatml_msgs;
// iterate the array
for (auto it = messages.begin(); it != messages.end(); ++it) { for (auto it = messages.begin(); it != messages.end(); ++it) {
chatml_msgs << "<|im_start|>" chatml_msgs << "<|im_start|>"
<< json_value(*it, "role", std::string("user")) << '\n'; << json_value(*it, "role", std::string("user")) << '\n';
@ -2246,52 +2245,36 @@ std::string format_chatml(std::vector<json> messages) {
/* llama.cpp completion api semantics */ /* llama.cpp completion api semantics */
json oaicompat_completion_params_parse( json oaicompat_completion_params_parse(
const json &body /* openai api json semantics */) { const json &body /* openai api json semantics */)
{
json llama_params; json llama_params;
llama_params["__oaicompat"] = true; llama_params["__oaicompat"] = true;
// Map OpenAI parameters to llama.cpp parameters // Map OpenAI parameters to llama.cpp parameters
llama_params["prompt"] = format_chatml( llama_params["prompt"] = format_chatml(body["messages"]); // OpenAI 'messages' to llama.cpp 'prompt'
body["messages"]); // OpenAI 'messages' to llama.cpp 'prompt' llama_params["temperature"] = json_value(body, "temperature", 0.8);
llama_params["temperature"] = llama_params["top_k"] = json_value(body, "max_tokens", 40);
json_value(body, "temperature", 0.8); // Default to 0.8 if not provided llama_params["top_p"] = json_value(body, "top_p", 0.95);
llama_params["top_k"] = llama_params["n_predict"] = json_value(body, "max_tokens", -1);
json_value(body, "max_tokens", 40); // Default to 40 if not provided llama_params["logit_bias"] = json_value(body, "logit_bias",json::object());
llama_params["top_p"] = llama_params["frequency_penalty"] = json_value(body, "frequency_penalty", 0.0);
json_value(body, "top_p", 0.95); // Default to 0.95 if not provided llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0);
llama_params["n_predict"] =
json_value(body, "max_tokens", -1); // Default to -1 if not provided
llama_params["logit_bias"] = json_value(
body, "logit_bias",
json::object()); // Default to empty object if not provided
llama_params["frequency_penalty"] = json_value(
body, "frequency_penalty", 0.0); // Default to 0.0 if not provided
llama_params["presence_penalty"] = json_value(
body, "presence_penalty", 0.0); // Default to 0.0 if not provided
llama_params["seed"] = json_value(body, "seed", 0); llama_params["seed"] = json_value(body, "seed", 0);
llama_params["stream"] = llama_params["stream"] =json_value(body, "stream", false);
json_value(body, "stream", false); // Default to 0 if not provided llama_params["mirostat"] = json_value(body, "mirostat", false);
llama_params["mirostat"] = llama_params["mirostat_tau"] = json_value(body, "mirostat_tau", 0.0);
json_value(body, "mirostat", false); // Default to false if not provided llama_params["mirostat_eta"] = json_value(body, "mirostat_eta", 0.0);
llama_params["mirostat_tau"] = llama_params["penalize_nl"] = json_value(body, "penalize_nl", false);
json_value(body, "mirostat_tau", 0.0); // Default to 0.0 if not provided llama_params["typical_p"] = json_value(body, "typical_p", 0.0);
llama_params["mirostat_eta"] = llama_params["repeat_last_n"] = json_value(body, "repeat_last_n", 0);
json_value(body, "mirostat_eta", 0.0); // Default to 0.0 if not provided llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);
llama_params["penalize_nl"] = json_value( llama_params["tfs_z"] = json_value(body, "tfs_z", 0.0);
body, "penalize_nl", false); // Default to false if not provided
llama_params["typical_p"] =
json_value(body, "typical_p", 0.0); // Default to 0.0 if not provided
llama_params["repeat_last_n"] =
json_value(body, "repeat_last_n", 0); // Default to 0 if not provided
llama_params["ignore_eos"] =
json_value(body, "ignore_eos", false); // Default to false if not provided
llama_params["tfs_z"] =
json_value(body, "tfs_z", 0.0); // Default to 0.0 if not provided
if (llama_params.count("grammar") != 0) { if (llama_params.count("grammar") != 0) {
llama_params["grammar"] = json_value( llama_params["grammar"] = json_value(
body, "grammar", body, "grammar",
json::object()); // Default to empty object if not provided json::object());
} }
// Handle 'stop' field // Handle 'stop' field
@ -2302,17 +2285,18 @@ json oaicompat_completion_params_parse(
} else { } else {
llama_params["stop"] = json_value( llama_params["stop"] = json_value(
body, "stop", body, "stop",
json::array()); // Default to empty array if not provided json::array());
} }
// Ensure there is ChatML-specific end sequence among stop words
llama_params["stop"].push_back("<|im_end|>"); llama_params["stop"].push_back("<|im_end|>");
return llama_params; return llama_params;
} }
static json format_final_response_oaicompat(json request, task_result response, static json format_final_response_oaicompat(json request, task_result response,
bool streaming = false) { bool streaming = false)
{
json result = response.result_json; json result = response.result_json;
bool stopped_word = result.count("stopped_word") != 0; bool stopped_word = result.count("stopped_word") != 0;
@ -2354,13 +2338,13 @@ static json format_final_response_oaicompat(json request, task_result response,
} }
if (result.contains("completion_probabilities")) { if (result.contains("completion_probabilities")) {
res["completion_probabilities"] = res["completion_probabilities"] = json_value(result, "completion_probabilities", json::array());
json_value(result, "completion_probabilities", json::array());
} }
return res; return res;
} }
// return value is vector as there is one case where we might need to generate two responses
static std::vector<json> format_partial_response_oaicompat(task_result response) { static std::vector<json> format_partial_response_oaicompat(task_result response) {
json result = response.result_json; json result = response.result_json;
@ -2401,8 +2385,7 @@ static std::vector<json> format_partial_response_oaicompat(task_result response)
{"delta", json{{"role", "assistant"}}}}}); {"delta", json{{"role", "assistant"}}}}});
} else { } else {
// We have to send this as two updates to conform to openai behavior // We have to send this as two updates to conform to openai behavior
json initial_ret = json{{"choices", json initial_ret = json{{"choices", json::array({json{
json::array({json{
{"finish_reason", nullptr}, {"finish_reason", nullptr},
{"index", 0}, {"index", 0},
{"delta", json{ {"delta", json{
@ -2413,24 +2396,26 @@ static std::vector<json> format_partial_response_oaicompat(task_result response)
{"model", modelname}, {"model", modelname},
{"object", "chat.completion.chunk"}}; {"object", "chat.completion.chunk"}};
json second_ret = json{{"choices", json second_ret = json{
json::array({json{ {"choices", json::array({json{{"finish_reason", nullptr},
{"finish_reason", nullptr},
{"index", 0}, {"index", 0},
{"delta", json{ {"delta", json{
{"content", content}}}}})}, {"content", content}}}
}})},
{"created", t}, {"created", t},
{"id", gen_chatcmplid()}, {"id", gen_chatcmplid()},
{"model", modelname}, {"model", modelname},
{"object", "chat.completion.chunk"}}; {"object", "chat.completion.chunk"}};
return std::vector<json>({initial_ret, second_ret}); return std::vector<json>({initial_ret, second_ret});
} }
} else { } else {
// Some idosyncrasy in task processing logic makes several trailing calls // Some idiosyncrasy in task processing logic makes several trailing calls
// with empty content, we ignore these at the calee site. // with empty content, we ignore these at the calee site.
if (content.empty()) { if (content.empty()) {
return std::vector<json>({json::object()}); return std::vector<json>({json::object()});
} }
choices = json::array({json{ choices = json::array({json{
{"finish_reason", nullptr}, {"finish_reason", nullptr},
{"index", 0}, {"index", 0},
@ -2671,10 +2656,12 @@ int main(int argc, char **argv)
svr.Post("/v1/chat/completions", [&llama](const httplib::Request &req, svr.Post("/v1/chat/completions", [&llama](const httplib::Request &req,
httplib::Response &res) { httplib::Response &res)
{
json data = oaicompat_completion_params_parse(json::parse(req.body)); json data = oaicompat_completion_params_parse(json::parse(req.body));
const int task_id = llama.request_completion(data, false, false); const int task_id = llama.request_completion(data, false, false);
if (!json_value(data, "stream", false)) { if (!json_value(data, "stream", false)) {
std::string completion_text; std::string completion_text;
task_result result = llama.next_result(task_id); task_result result = llama.next_result(task_id);
@ -2691,8 +2678,7 @@ int main(int argc, char **argv)
return; return;
} }
} else { } else {
const auto chunked_content_provider = [task_id, &llama](size_t, const auto chunked_content_provider = [task_id, &llama](size_t, httplib::DataSink &sink) {
httplib::DataSink &sink) {
while (true) { while (true) {
task_result llama_result = llama.next_result(task_id); task_result llama_result = llama.next_result(task_id);
if (!llama_result.error) { if (!llama_result.error) {
@ -2732,12 +2718,11 @@ int main(int argc, char **argv)
}; };
auto on_complete = [task_id, &llama](bool) { auto on_complete = [task_id, &llama](bool) {
// cancel // cancel request
llama.request_cancel(task_id); llama.request_cancel(task_id);
}; };
res.set_chunked_content_provider("text/event-stream", res.set_chunked_content_provider("text/event-stream", chunked_content_provider, on_complete);
chunked_content_provider, on_complete);
} }
}); });