Mirrors/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-27 06:39:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

server : fix slot selection by lru (#10126)

Browse Source 
* server : fix slot selection by lru, migrate lcs to `size_t`

* minor debug log fix
This commit is contained in:
sasha0552 2024-11-02 16:34:56 +00:00 committed by GitHub
parent 45950415ed
commit 42cadc74bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
examples/server/server.cpp
View File

@ -247,6 +247,7 @@ struct server_slot {
if (is_processing()) { if (is_processing()) {
SLT_INF(*this, "stop processing: n_past = %d, truncated = %d\n", n_past, truncated); SLT_INF(*this, "stop processing: n_past = %d, truncated = %d\n", n_past, truncated);
t_last_used = ggml_time_us();
t_token_generation = (ggml_time_us() - t_start_generation) / 1e3; t_token_generation = (ggml_time_us() - t_start_generation) / 1e3;
state = SLOT_STATE_IDLE; state = SLOT_STATE_IDLE;
callback_on_release(id); callback_on_release(id);
@ -730,7 +731,7 @@ struct server_context {
// find the slot that has at least n% prompt similarity // find the slot that has at least n% prompt similarity
if (ret == nullptr && slot_prompt_similarity != 0.0f) { if (ret == nullptr && slot_prompt_similarity != 0.0f) {
int max_lcs_len = 0; int lcs_len = 0;
float similarity = 0; float similarity = 0;
for (server_slot & slot : slots) { for (server_slot & slot : slots) {
@ -745,20 +746,21 @@ struct server_context {
} }
// length of the Longest Common Subsequence between the current slot's prompt and the input prompt // length of the Longest Common Subsequence between the current slot's prompt and the input prompt
int lcs_len = longest_common_subsequence(slot.cache_tokens, task.prompt_tokens); int cur_lcs_len = longest_common_subsequence(slot.cache_tokens, task.prompt_tokens);
// fraction of the common subsequence length compared to the current slot's prompt length // fraction of the common subsequence length compared to the current slot's prompt length
similarity = static_cast<float>(lcs_len) / static_cast<int>(slot.cache_tokens.size()); float cur_similarity = static_cast<float>(cur_lcs_len) / static_cast<int>(slot.cache_tokens.size());
// select the current slot if the criteria match // select the current slot if the criteria match
if (lcs_len > max_lcs_len && similarity > slot_prompt_similarity) { if (cur_lcs_len > lcs_len && cur_similarity > slot_prompt_similarity) {
max_lcs_len = lcs_len; lcs_len = cur_lcs_len;
similarity = cur_similarity;
ret = &slot; ret = &slot;
} }
} }
if (ret != nullptr) { if (ret != nullptr) {
SLT_DBG(*ret, "selected slot by lcs similarity, max_lcs_len = %d, similarity = %f\n", max_lcs_len, similarity); SLT_DBG(*ret, "selected slot by lcs similarity, lcs_len = %d, similarity = %f\n", lcs_len, similarity);
} }
} }

14
examples/server/utils.hpp
View File

@ -453,20 +453,20 @@ static size_t longest_common_subsequence(const llama_tokens & a, const llama_tok
} }
// get the lengths of the input sequences // get the lengths of the input sequences
int a_len = a.size(); size_t a_len = a.size();
int b_len = b.size(); size_t b_len = b.size();
// initialize the maximum length of the longest common subsequence (LCS) // initialize the maximum length of the longest common subsequence (LCS)
int max_length = 0; size_t max_length = 0;
// use two rows instead of a 2D matrix to optimize space // use two rows instead of a 2D matrix to optimize space
std::vector<int> prev_row(b_len + 1, 0); std::vector<size_t> prev_row(b_len + 1, 0);
std::vector<int> curr_row(b_len + 1, 0); std::vector<size_t> curr_row(b_len + 1, 0);
// iterate through the elements of a // iterate through the elements of a
for (int i = 1; i <= a_len; i++) { for (size_t i = 1; i <= a_len; i++) {
// iterate through the elements of b // iterate through the elements of b
for (int j = 1; j <= b_len; j++) { for (size_t j = 1; j <= b_len; j++) {
// if elements at the current positions match // if elements at the current positions match
if (a[i - 1] == b[j - 1]) { if (a[i - 1] == b[j - 1]) {
// if it's the first element of either sequences, set LCS length to 1 // if it's the first element of either sequences, set LCS length to 1