From c1b738ef43540e7102a14abe38ce894be9ac803c Mon Sep 17 00:00:00 2001 From: Francis Couture-Harpin Date: Tue, 13 Aug 2024 22:03:57 -0400 Subject: [PATCH] server : fix parallel generation with very small batch sizes --- examples/server/server.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index a1bbafbc9..b2bf5bb5f 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -753,13 +753,13 @@ struct server_context { default_generation_settings_for_props = get_formated_generation(slots.front()); default_generation_settings_for_props["seed"] = -1; - // the update_slots() logic will always submit a maximum of n_batch tokens + // the update_slots() logic will always submit a maximum of n_batch or n_parralel tokens // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used) { const int32_t n_batch = llama_n_batch(ctx); // only a single seq_id per token is needed - batch = llama_batch_init(n_batch, 0, 1); + batch = llama_batch_init(std::max(n_batch, params.n_parallel), 0, 1); } metrics.init();