-
+
@@ -30,7 +32,7 @@
@@ -40,7 +42,7 @@
} options
+ * @param {*} defaultOption
+ * @param {function(string):void} cb
+ */
+export function el_create_select(id, options, defaultOption, cb) {
+ let el = document.createElement("select");
+ el["xselected"] = defaultOption;
+ el["xoptions"] = structuredClone(options);
+ for(let cur of Object.keys(options)) {
+ let op = document.createElement("option");
+ op.value = cur;
+ op.innerText = cur;
+ if (options[cur] == defaultOption) {
+ op.selected = true;
+ }
+ el.appendChild(op);
+ }
+ if (id) {
+ el.id = id;
+ el.name = id;
+ }
+ el.addEventListener('change', (ev)=>{
+ let target = /** @type{HTMLSelectElement} */(ev.target);
+ console.log("DBUG:UI:Select:", id, ":", target.value);
+ cb(target.value);
+ })
+ return el;
+}
+
+/**
+ * Create a div wrapped select ui element, with a set of options to select from.
+ *
+ * @param {string} id
+ * @param {any} label
+ * @param {{ [x: string]: any; }} options
+ * @param {any} defaultOption
+ * @param {(arg0: string) => void} cb
+ * @param {string} className
+ */
+export function el_creatediv_select(id, label, options, defaultOption, cb, className="gridx2") {
+ let div = document.createElement("div");
+ div.className = className;
+ let lbl = document.createElement("label");
+ lbl.setAttribute("for", id);
+ lbl.innerText = label;
+ div.appendChild(lbl);
+ let sel = el_create_select(id, options,defaultOption, cb);
+ div.appendChild(sel);
+ return { div: div, el: sel };
+}
+
+
+/**
+ * Create a input ui element.
+ *
+ * @param {string} id
+ * @param {string} type
+ * @param {any} defaultValue
+ * @param {function(any):void} cb
+ */
+export function el_create_input(id, type, defaultValue, cb) {
+ let el = document.createElement("input");
+ el.type = type;
+ el.value = defaultValue;
+ if (id) {
+ el.id = id;
+ }
+ el.addEventListener('change', (ev)=>{
+ cb(el.value);
+ })
+ return el;
+}
+
+/**
+ * Create a div wrapped input.
+ *
+ * @param {string} id
+ * @param {string} label
+ * @param {string} type
+ * @param {any} defaultValue
+ * @param {function(any):void} cb
+ * @param {string} className
+ */
+export function el_creatediv_input(id, label, type, defaultValue, cb, className="gridx2") {
+ let div = document.createElement("div");
+ div.className = className;
+ let lbl = document.createElement("label");
+ lbl.setAttribute("for", id);
+ lbl.innerText = label;
+ div.appendChild(lbl);
+ let el = el_create_input(id, type, defaultValue, cb);
+ div.appendChild(el);
+ return { div: div, el: el };
+}
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index e9904263d..6ffaa8d9f 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -17,9 +17,20 @@
#include "json.hpp"
// auto generated files (update with ./deps.sh)
+#include "colorthemes.css.hpp"
+#include "style.css.hpp"
+#include "theme-beeninorder.css.hpp"
+#include "theme-ketivah.css.hpp"
+#include "theme-mangotango.css.hpp"
+#include "theme-playground.css.hpp"
+#include "theme-polarnight.css.hpp"
+#include "theme-snowstorm.css.hpp"
#include "index.html.hpp"
+#include "index-new.html.hpp"
#include "index.js.hpp"
#include "completion.js.hpp"
+#include "system-prompts.js.hpp"
+#include "prompt-formats.js.hpp"
#include "json-schema-to-grammar.mjs.hpp"
#include
@@ -112,29 +123,6 @@ struct slot_params {
json input_suffix;
};
-struct server_params {
- int32_t port = 8080;
- int32_t read_timeout = 600;
- int32_t write_timeout = 600;
- int32_t n_threads_http = -1;
-
- std::string hostname = "127.0.0.1";
- std::string public_path = "";
- std::string chat_template = "";
- std::string system_prompt = "";
-
- std::vector api_keys;
-
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- std::string ssl_key_file = "";
- std::string ssl_cert_file = "";
-#endif
-
- bool slots_endpoint = true;
- bool metrics_endpoint = false;
- std::string slot_save_path;
-};
-
struct server_slot {
int id;
int id_task = -1;
@@ -659,6 +647,9 @@ struct server_context {
server_metrics metrics;
+ // Necessary similarity of prompt for slot selection
+ float slot_prompt_similarity = 0.0f;
+
~server_context() {
if (ctx) {
llama_free(ctx);
@@ -807,24 +798,88 @@ struct server_context {
return prompt_tokens;
}
- server_slot * get_slot(int id) {
- int64_t t_last = ggml_time_us();
-
- server_slot * last_used = nullptr;
-
+ server_slot * get_slot_by_id(int id) {
for (server_slot & slot : slots) {
- if (slot.id == id && slot.available()) {
+ if (slot.id == id) {
return &slot;
}
-
- // among all available slots, find the one that has been least recently used
- if (slot.available() && slot.t_last_used < t_last) {
- last_used = &slot;
- t_last = slot.t_last_used;
- }
}
- return last_used;
+ return nullptr;
+ }
+
+ server_slot * get_available_slot(const std::string & prompt) {
+ server_slot * ret = nullptr;
+
+ // find the slot that has at least n% prompt similarity
+ if (ret == nullptr && slot_prompt_similarity != 0.0f && !prompt.empty()) {
+ int max_lcp_len = 0;
+ float similarity = 0;
+
+ for (server_slot & slot : slots) {
+ // skip the slot if it is not available
+ if (!slot.available()) {
+ continue;
+ }
+
+ // skip the slot if it does not contains prompt
+ if (!slot.prompt.is_string()) {
+ continue;
+ }
+
+ // current slot's prompt
+ std::string slot_prompt = slot.prompt.get();
+
+ // length of the current slot's prompt
+ int slot_prompt_len = slot_prompt.size();
+
+ // length of the Longest Common Prefix between the current slot's prompt and the input prompt
+ int lcp_len = common_part(slot_prompt, prompt);
+
+ // fraction of the common substring length compared to the current slot's prompt length
+ similarity = static_cast(lcp_len) / slot_prompt_len;
+
+ // select the current slot if the criteria match
+ if (lcp_len > max_lcp_len && similarity > slot_prompt_similarity) {
+ max_lcp_len = lcp_len;
+ ret = &slot;
+ }
+ }
+
+ if (ret != nullptr) {
+ LOG_VERBOSE("selected slot by lcp similarity", {
+ {"id_slot", ret->id},
+ {"max_lcp_len", max_lcp_len},
+ {"similarity", similarity},
+ });
+ }
+ }
+
+ // find the slot that has been least recently used
+ if (ret == nullptr) {
+ int64_t t_last = ggml_time_us();
+ for (server_slot & slot : slots) {
+ // skip the slot if it is not available
+ if (!slot.available()) {
+ continue;
+ }
+
+ // select the current slot if the criteria match
+ if (slot.t_last_used < t_last) {
+ t_last = slot.t_last_used;
+ ret = &slot;
+ }
+ }
+
+ if (ret != nullptr) {
+ LOG_VERBOSE("selected slot by lru", {
+ {"id_slot", ret->id},
+ {"t_last", t_last},
+ });
+ }
+ }
+
+ return ret;
}
bool launch_slot_with_task(server_slot & slot, const server_task & task) {
@@ -900,7 +955,7 @@ struct server_context {
slot.params.input_suffix = json_value(data, "input_suffix", default_params.input_suffix);
// get prompt
- {
+ if (!task.infill) {
const auto & prompt = data.find("prompt");
if (prompt == data.end()) {
send_error(task, "Either \"prompt\" or \"messages\" must be provided", ERROR_TYPE_INVALID_REQUEST);
@@ -1250,7 +1305,7 @@ struct server_context {
}
json get_formated_generation(const server_slot & slot) const {
- const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));
+ const auto eos_bias = slot.sparams.logit_bias.find(llama_token_eos(model));
const bool ignore_eos = eos_bias != slot.sparams.logit_bias.end() && eos_bias->second < 0.0f && std::isinf(eos_bias->second);
std::vector samplers_sequence;
@@ -1527,13 +1582,29 @@ struct server_context {
switch (task.type) {
case SERVER_TASK_TYPE_COMPLETION:
{
- server_slot * slot = get_slot(json_value(task.data, "id_slot", -1));
+ int id_slot = json_value(task.data, "id_slot", -1);
+ std::string prompt = json_value(task.data, "prompt", std::string());
+
+ server_slot * slot;
+
+ if (id_slot != -1) {
+ slot = get_slot_by_id(id_slot);
+ } else {
+ slot = get_available_slot(prompt);
+ }
+
if (slot == nullptr) {
// if no slot is available, we defer this task for processing later
LOG_VERBOSE("no slot is available", {{"id_task", task.id}});
queue_tasks.defer(task);
break;
}
+ if (!slot->available()) {
+ // if requested slot is unavailable, we defer this task for processing later
+ LOG_VERBOSE("requested slot is unavailable", {{"id_task", task.id}});
+ queue_tasks.defer(task);
+ break;
+ }
if (task.data.contains("system_prompt")) {
std::string sys_prompt = json_value(task.data, "system_prompt", std::string());
@@ -1650,11 +1721,17 @@ struct server_context {
case SERVER_TASK_TYPE_SLOT_SAVE:
{
int id_slot = task.data.at("id_slot");
- server_slot * slot = get_slot(id_slot);
+ server_slot * slot = get_slot_by_id(id_slot);
if (slot == nullptr) {
send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST);
break;
}
+ if (!slot->available()) {
+ // if requested slot is unavailable, we defer this task for processing later
+ LOG_VERBOSE("requested slot is unavailable", {{"id_task", task.id}});
+ queue_tasks.defer(task);
+ break;
+ }
const size_t token_count = slot->cache_tokens.size();
const int64_t t_start = ggml_time_us();
@@ -1685,11 +1762,17 @@ struct server_context {
case SERVER_TASK_TYPE_SLOT_RESTORE:
{
int id_slot = task.data.at("id_slot");
- server_slot * slot = get_slot(id_slot);
+ server_slot * slot = get_slot_by_id(id_slot);
if (slot == nullptr) {
send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST);
break;
}
+ if (!slot->available()) {
+ // if requested slot is unavailable, we defer this task for processing later
+ LOG_VERBOSE("requested slot is unavailable", {{"id_task", task.id}});
+ queue_tasks.defer(task);
+ break;
+ }
const int64_t t_start = ggml_time_us();
@@ -1727,11 +1810,17 @@ struct server_context {
case SERVER_TASK_TYPE_SLOT_ERASE:
{
int id_slot = task.data.at("id_slot");
- server_slot * slot = get_slot(id_slot);
+ server_slot * slot = get_slot_by_id(id_slot);
if (slot == nullptr) {
send_error(task, "Invalid slot ID", ERROR_TYPE_INVALID_REQUEST);
break;
}
+ if (!slot->available()) {
+ // if requested slot is unavailable, we defer this task for processing later
+ LOG_VERBOSE("requested slot is unavailable", {{"id_task", task.id}});
+ queue_tasks.defer(task);
+ break;
+ }
// Erase token cache
const size_t n_erased = slot->cache_tokens.size();
@@ -2323,561 +2412,6 @@ struct server_context {
}
};
-static void server_print_usage(const char * argv0, const gpt_params & params, const server_params & sparams) {
- printf("usage: %s [options]\n", argv0);
- printf("\n");
- printf("options:\n");
- printf(" -h, --help show this help message and exit\n");
- printf(" -v, --verbose verbose output (default: %s)\n", server_verbose ? "enabled" : "disabled");
- printf(" -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
- printf(" -tb N, --threads-batch N number of threads to use during batch and prompt processing (default: same as --threads)\n");
- printf(" --threads-http N number of threads in the http server pool to process requests (default: max(hardware concurrency - 1, --parallel N + 2))\n");
- printf(" -c N, --ctx-size N size of the prompt context (default: %d)\n", params.n_ctx);
- printf(" --rope-scaling {none,linear,yarn}\n");
- printf(" RoPE frequency scaling method, defaults to linear unless specified by the model\n");
- printf(" --rope-freq-base N RoPE base frequency (default: loaded from model)\n");
- printf(" --rope-freq-scale N RoPE frequency scaling factor, expands context by a factor of 1/N\n");
- printf(" --yarn-ext-factor N YaRN: extrapolation mix factor (default: 1.0, 0.0 = full interpolation)\n");
- printf(" --yarn-attn-factor N YaRN: scale sqrt(t) or attention magnitude (default: 1.0)\n");
- printf(" --yarn-beta-slow N YaRN: high correction dim or alpha (default: %.1f)\n", params.yarn_beta_slow);
- printf(" --yarn-beta-fast N YaRN: low correction dim or beta (default: %.1f)\n", params.yarn_beta_fast);
- printf(" --pooling {none,mean,cls} pooling type for embeddings, use model default if unspecified\n");
- printf(" -dt N, --defrag-thold N\n");
- printf(" KV cache defragmentation threshold (default: %.1f, < 0 - disabled)\n", params.defrag_thold);
- printf(" -b N, --batch-size N logical maximum batch size (default: %d)\n", params.n_batch);
- printf(" -ub N, --ubatch-size N physical maximum batch size (default: %d)\n", params.n_ubatch);
- if (llama_supports_mlock()) {
- printf(" --mlock force system to keep model in RAM rather than swapping or compressing\n");
- }
- if (llama_supports_mmap()) {
- printf(" --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");
- }
- printf(" --numa TYPE attempt optimizations that help on some NUMA systems\n");
- printf(" - distribute: spread execution evenly over all nodes\n");
- printf(" - isolate: only spawn threads on CPUs on the node that execution started on\n");
- printf(" - numactl: use the CPU map provided my numactl\n");
- if (llama_supports_gpu_offload()) {
- printf(" -ngl N, --n-gpu-layers N\n");
- printf(" number of layers to store in VRAM\n");
- printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");
- printf(" how to split the model across multiple GPUs, one of:\n");
- printf(" - none: use one GPU only\n");
- printf(" - layer (default): split layers and KV across GPUs\n");
- printf(" - row: split rows across GPUs\n");
- printf(" -ts SPLIT --tensor-split SPLIT\n");
- printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");
- printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");
- printf(" or for intermediate results and KV (with split-mode = row)\n");
- printf(" -nkvo, --no-kv-offload\n");
- printf(" disable KV offload\n");
- }
- printf(" -m FNAME, --model FNAME\n");
- printf(" model path (default: models/$filename with filename from --hf-file or --model-url if set, otherwise %s)\n", DEFAULT_MODEL_PATH);
- printf(" -mu MODEL_URL, --model-url MODEL_URL\n");
- printf(" model download url (default: unused)\n");
- printf(" -hfr REPO, --hf-repo REPO\n");
- printf(" Hugging Face model repository (default: unused)\n");
- printf(" -hff FILE, --hf-file FILE\n");
- printf(" Hugging Face model file (default: unused)\n");
- printf(" -a ALIAS, --alias ALIAS\n");
- printf(" set an alias for the model, will be added as `model` field in completion response\n");
- printf(" --lora FNAME apply LoRA adapter (implies --no-mmap)\n");
- printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");
- printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());
- printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);
- printf(" --rpc SERVERS comma separated list of RPC servers\n");
- printf(" --path PUBLIC_PATH path from which to serve static files (default: disabled)\n");
- printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");
- printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- printf(" --ssl-key-file FNAME path to file a PEM-encoded SSL private key\n");
- printf(" --ssl-cert-file FNAME path to file a PEM-encoded SSL certificate\n");
-#endif
- printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);
- printf(" --embeddings enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");
- printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);
- printf(" -cb, --cont-batching enable continuous batching (a.k.a dynamic batching) (default: enabled)\n");
- printf(" -fa, --flash-attn enable Flash Attention (default: %s)\n", params.flash_attn ? "enabled" : "disabled");
- printf(" -spf FNAME, --system-prompt-file FNAME\n");
- printf(" set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.\n");
- printf(" -ctk TYPE, --cache-type-k TYPE\n");
- printf(" KV cache data type for K (default: f16)\n");
- printf(" -ctv TYPE, --cache-type-v TYPE\n");
- printf(" KV cache data type for V (default: f16)\n");
- printf(" --log-format log output format: json or text (default: json)\n");
- printf(" --log-disable disables logging to a file.\n");
- printf(" --slots-endpoint-disable disables slots monitoring endpoint.\n");
- printf(" --metrics enable prometheus compatible metrics endpoint (default: %s).\n", sparams.metrics_endpoint ? "enabled" : "disabled");
- printf(" --slot-save-path PATH path to save slot kv cache (default: disabled)\n");
- printf("\n");
- printf(" -n, --n-predict maximum tokens to predict (default: %d)\n", params.n_predict);
- printf(" --override-kv KEY=TYPE:VALUE\n");
- printf(" advanced option to override model metadata by key. may be specified multiple times.\n");
- printf(" types: int, float, bool, str. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");
- printf(" -gan N, --grp-attn-n N set the group attention factor to extend context size through self-extend(default: 1=disabled), used together with group attention width `--grp-attn-w`\n");
- printf(" -gaw N, --grp-attn-w N set the group attention width to extend context size through self-extend(default: 512), used together with group attention factor `--grp-attn-n`\n");
- printf(" --chat-template JINJA_TEMPLATE\n");
- printf(" set custom jinja chat template (default: template taken from model's metadata)\n");
- printf(" only commonly used templates are accepted:\n");
- printf(" https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template\n");
- printf("\n");
-}
-
-static void server_params_parse(int argc, char ** argv, server_params & sparams, gpt_params & params) {
- gpt_params default_params;
- server_params default_sparams;
-
- std::string arg;
- bool invalid_param = false;
-
- for (int i = 1; i < argc; i++) {
- arg = argv[i];
- if (arg == "--port") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.port = std::stoi(argv[i]);
- } else if (arg == "--rpc") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.rpc_servers = argv[i];
- } else if (arg == "--host") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.hostname = argv[i];
- } else if (arg == "--path") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.public_path = argv[i];
- } else if (arg == "--api-key") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.api_keys.push_back(argv[i]);
- } else if (arg == "--api-key-file") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- std::ifstream key_file(argv[i]);
- if (!key_file) {
- fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
- invalid_param = true;
- break;
- }
- std::string key;
- while (std::getline(key_file, key)) {
- if (key.size() > 0) {
- sparams.api_keys.push_back(key);
- }
- }
- key_file.close();
-
- }
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- else if (arg == "--ssl-key-file") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.ssl_key_file = argv[i];
- } else if (arg == "--ssl-cert-file") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.ssl_cert_file = argv[i];
- }
-#endif
- else if (arg == "--timeout" || arg == "-to") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.read_timeout = std::stoi(argv[i]);
- sparams.write_timeout = std::stoi(argv[i]);
- } else if (arg == "-m" || arg == "--model") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.model = argv[i];
- } else if (arg == "-mu" || arg == "--model-url") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.model_url = argv[i];
- } else if (arg == "-hfr" || arg == "--hf-repo") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.hf_repo = argv[i];
- } else if (arg == "-hff" || arg == "--hf-file") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.hf_file = argv[i];
- } else if (arg == "-a" || arg == "--alias") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.model_alias = argv[i];
- } else if (arg == "-h" || arg == "--help") {
- server_print_usage(argv[0], default_params, default_sparams);
- exit(0);
- } else if (arg == "-c" || arg == "--ctx-size" || arg == "--ctx_size") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.n_ctx = std::stoi(argv[i]);
- } else if (arg == "--rope-scaling") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- std::string value(argv[i]);
- /**/ if (value == "none") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE; }
- else if (value == "linear") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_LINEAR; }
- else if (value == "yarn") { params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_YARN; }
- else { invalid_param = true; break; }
- } else if (arg == "--rope-freq-base") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.rope_freq_base = std::stof(argv[i]);
- } else if (arg == "--rope-freq-scale") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.rope_freq_scale = std::stof(argv[i]);
- } else if (arg == "--yarn-ext-factor") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.yarn_ext_factor = std::stof(argv[i]);
- }
- else if (arg == "--yarn-attn-factor") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.yarn_attn_factor = std::stof(argv[i]);
- } else if (arg == "--yarn-beta-fast") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.yarn_beta_fast = std::stof(argv[i]);
- } else if (arg == "--yarn-beta-slow") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.yarn_beta_slow = std::stof(argv[i]);
- } else if (arg == "--pooling") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- std::string value(argv[i]);
- /**/ if (value == "none") { params.pooling_type = LLAMA_POOLING_TYPE_NONE; }
- else if (value == "mean") { params.pooling_type = LLAMA_POOLING_TYPE_MEAN; }
- else if (value == "cls") { params.pooling_type = LLAMA_POOLING_TYPE_CLS; }
- else { invalid_param = true; break; }
- } else if (arg == "--defrag-thold" || arg == "-dt") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.defrag_thold = std::stof(argv[i]);
- } else if (arg == "--threads" || arg == "-t") {
- if (++i >= argc)
- {
- invalid_param = true;
- break;
- }
- params.n_threads = std::stoi(argv[i]);
- } else if (arg == "--grp-attn-n" || arg == "-gan") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
-
- params.grp_attn_n = std::stoi(argv[i]);
- } else if (arg == "--grp-attn-w" || arg == "-gaw") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
-
- params.grp_attn_w = std::stoi(argv[i]);
- } else if (arg == "--threads-batch" || arg == "-tb") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.n_threads_batch = std::stoi(argv[i]);
- } else if (arg == "--threads-http") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.n_threads_http = std::stoi(argv[i]);
- } else if (arg == "-b" || arg == "--batch-size") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.n_batch = std::stoi(argv[i]);
- } else if (arg == "-ub" || arg == "--ubatch-size") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.n_ubatch = std::stoi(argv[i]);
- } else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- if (llama_supports_gpu_offload()) {
- params.n_gpu_layers = std::stoi(argv[i]);
- } else {
- LOG_WARNING(
- "Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "
- "See main README.md for information on enabling GPU BLAS support",
- {{"n_gpu_layers", params.n_gpu_layers}});
- }
- } else if (arg == "-nkvo" || arg == "--no-kv-offload") {
- params.no_kv_offload = true;
- } else if (arg == "--split-mode" || arg == "-sm") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- std::string arg_next = argv[i];
- if (arg_next == "none") {
- params.split_mode = LLAMA_SPLIT_MODE_NONE;
- } else if (arg_next == "layer") {
- params.split_mode = LLAMA_SPLIT_MODE_LAYER;
- } else if (arg_next == "row") {
- params.split_mode = LLAMA_SPLIT_MODE_ROW;
- } else {
- invalid_param = true;
- break;
- }
-#ifndef GGML_USE_CUDA
- fprintf(stderr, "warning: llama.cpp was compiled without CUDA. Setting the split mode has no effect.\n");
-#endif // GGML_USE_CUDA
- } else if (arg == "--tensor-split" || arg == "-ts") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
-#if defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL)
- std::string arg_next = argv[i];
-
- // split string by , and /
- const std::regex regex{R"([,/]+)"};
- std::sregex_token_iterator it{arg_next.begin(), arg_next.end(), regex, -1};
- std::vector split_arg{it, {}};
- GGML_ASSERT(split_arg.size() <= llama_max_devices());
-
- for (size_t i_device = 0; i_device < llama_max_devices(); ++i_device) {
- if (i_device < split_arg.size()) {
- params.tensor_split[i_device] = std::stof(split_arg[i_device]);
- } else {
- params.tensor_split[i_device] = 0.0f;
- }
- }
-#else
- LOG_WARNING("llama.cpp was compiled without CUDA. It is not possible to set a tensor split.\n", {});
-#endif // GGML_USE_CUDA
- } else if (arg == "--main-gpu" || arg == "-mg") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
-#if defined(GGML_USE_CUDA) || defined(GGML_USE_SYCL)
- params.main_gpu = std::stoi(argv[i]);
-#else
- LOG_WARNING("llama.cpp was compiled without CUDA. It is not possible to set a main GPU.", {});
-#endif
- } else if (arg == "--lora") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.lora_adapter.emplace_back(argv[i], 1.0f);
- params.use_mmap = false;
- } else if (arg == "--lora-scaled") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- const char * lora_adapter = argv[i];
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.lora_adapter.emplace_back(lora_adapter, std::stof(argv[i]));
- params.use_mmap = false;
- } else if (arg == "--lora-base") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.lora_base = argv[i];
- } else if (arg == "-v" || arg == "--verbose") {
-#if SERVER_VERBOSE != 1
- LOG_WARNING("server.cpp is not built with verbose logging.", {});
-#else
- server_verbose = true;
-#endif
- } else if (arg == "--mlock") {
- params.use_mlock = true;
- } else if (arg == "--no-mmap") {
- params.use_mmap = false;
- } else if (arg == "--numa") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- } else {
- std::string value(argv[i]);
- /**/ if (value == "distribute" || value == "" ) { params.numa = GGML_NUMA_STRATEGY_DISTRIBUTE; }
- else if (value == "isolate") { params.numa = GGML_NUMA_STRATEGY_ISOLATE; }
- else if (value == "numactl") { params.numa = GGML_NUMA_STRATEGY_NUMACTL; }
- else { invalid_param = true; break; }
- }
- } else if (arg == "--embedding" || arg == "--embeddings") {
- params.embedding = true;
- } else if (arg == "-cb" || arg == "--cont-batching") {
- params.cont_batching = true;
- } else if (arg == "-fa" || arg == "--flash-attn") {
- params.flash_attn = true;
- } else if (arg == "-np" || arg == "--parallel") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.n_parallel = std::stoi(argv[i]);
- } else if (arg == "-n" || arg == "--n-predict") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- params.n_predict = std::stoi(argv[i]);
- } else if (arg == "-spf" || arg == "--system-prompt-file") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- std::ifstream file(argv[i]);
- if (!file) {
- fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
- invalid_param = true;
- break;
- }
- std::string system_prompt;
- std::copy(
- std::istreambuf_iterator(file),
- std::istreambuf_iterator(),
- std::back_inserter(system_prompt)
- );
- sparams.system_prompt = system_prompt;
- } else if (arg == "-ctk" || arg == "--cache-type-k") {
- params.cache_type_k = argv[++i];
- } else if (arg == "-ctv" || arg == "--cache-type-v") {
- params.cache_type_v = argv[++i];
- } else if (arg == "--log-format") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- if (std::strcmp(argv[i], "json") == 0) {
- server_log_json = true;
- } else if (std::strcmp(argv[i], "text") == 0) {
- server_log_json = false;
- } else {
- invalid_param = true;
- break;
- }
- } else if (arg == "--log-disable") {
- log_set_target(stdout);
- LOG_INFO("logging to file is disabled.", {});
- } else if (arg == "--slots-endpoint-disable") {
- sparams.slots_endpoint = false;
- } else if (arg == "--metrics") {
- sparams.metrics_endpoint = true;
- } else if (arg == "--slot-save-path") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- sparams.slot_save_path = argv[i];
- // if doesn't end with DIRECTORY_SEPARATOR, add it
- if (!sparams.slot_save_path.empty() && sparams.slot_save_path[sparams.slot_save_path.size() - 1] != DIRECTORY_SEPARATOR) {
- sparams.slot_save_path += DIRECTORY_SEPARATOR;
- }
- } else if (arg == "--chat-template") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- if (!verify_custom_template(argv[i])) {
- fprintf(stderr, "error: the supplied chat template is not supported: %s\n", argv[i]);
- fprintf(stderr, "note: llama.cpp does not use jinja parser, we only support commonly used templates\n");
- invalid_param = true;
- break;
- }
- sparams.chat_template = argv[i];
- } else if (arg == "--override-kv") {
- if (++i >= argc) {
- invalid_param = true;
- break;
- }
- if (!string_parse_kv_override(argv[i], params.kv_overrides)) {
- fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);
- invalid_param = true;
- break;
- }
- } else {
- fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
- server_print_usage(argv[0], default_params, default_sparams);
- exit(1);
- }
- }
-
- gpt_params_handle_model_default(params);
-
- if (!params.kv_overrides.empty()) {
- params.kv_overrides.emplace_back();
- params.kv_overrides.back().key[0] = 0;
- }
-
- if (invalid_param) {
- fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
- server_print_usage(argv[0], default_params, default_sparams);
- exit(1);
- }
-}
-
static void log_server_request(const httplib::Request & req, const httplib::Response & res) {
// skip GH copilot requests when using default port
if (req.path == "/v1/health" || req.path == "/v1/completions") {
@@ -2918,16 +2452,22 @@ int main(int argc, char ** argv) {
log_disable();
#endif
// own arguments required by this example
- gpt_params params;
- server_params sparams;
+ gpt_params params;
+
+ if (!gpt_params_parse(argc, argv, params)) {
+ gpt_params_print_usage(argc, argv, params);
+ return 1;
+ }
+
+ // TODO: not great to use extern vars
+ server_log_json = params.log_json;
+ server_verbose = params.verbosity > 0;
// struct that contains llama context and inference
server_context ctx_server;
- server_params_parse(argc, argv, sparams, params);
-
- if (!sparams.system_prompt.empty()) {
- ctx_server.system_prompt_set(sparams.system_prompt);
+ if (!params.system_prompt.empty()) {
+ ctx_server.system_prompt_set(params.system_prompt);
}
if (params.model_alias == "unknown") {
@@ -2951,10 +2491,10 @@ int main(int argc, char ** argv) {
std::unique_ptr svr;
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
- if (sparams.ssl_key_file != "" && sparams.ssl_cert_file != "") {
- LOG_INFO("Running with SSL", {{"key", sparams.ssl_key_file}, {"cert", sparams.ssl_cert_file}});
+ if (params.ssl_file_key != "" && params.ssl_file_cert != "") {
+ LOG_INFO("Running with SSL", {{"key", params.ssl_file_key}, {"cert", params.ssl_file_cert}});
svr.reset(
- new httplib::SSLServer(sparams.ssl_cert_file.c_str(), sparams.ssl_key_file.c_str())
+ new httplib::SSLServer(params.ssl_file_cert.c_str(), params.ssl_file_key.c_str())
);
} else {
LOG_INFO("Running without SSL", {});
@@ -3008,26 +2548,29 @@ int main(int argc, char ** argv) {
});
// set timeouts and change hostname and port
- svr->set_read_timeout (sparams.read_timeout);
- svr->set_write_timeout(sparams.write_timeout);
+ svr->set_read_timeout (params.timeout_read);
+ svr->set_write_timeout(params.timeout_write);
- if (!svr->bind_to_port(sparams.hostname, sparams.port)) {
- fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", sparams.hostname.c_str(), sparams.port);
+ if (!svr->bind_to_port(params.hostname, params.port)) {
+ fprintf(stderr, "\ncouldn't bind to server socket: hostname=%s port=%d\n\n", params.hostname.c_str(), params.port);
return 1;
}
std::unordered_map log_data;
- log_data["hostname"] = sparams.hostname;
- log_data["port"] = std::to_string(sparams.port);
+ log_data["hostname"] = params.hostname;
+ log_data["port"] = std::to_string(params.port);
- if (sparams.api_keys.size() == 1) {
- auto key = sparams.api_keys[0];
+ if (params.api_keys.size() == 1) {
+ auto key = params.api_keys[0];
log_data["api_key"] = "api_key: ****" + key.substr(std::max((int)(key.length() - 4), 0));
- } else if (sparams.api_keys.size() > 1) {
- log_data["api_key"] = "api_key: " + std::to_string(sparams.api_keys.size()) + " keys loaded";
+ } else if (params.api_keys.size() > 1) {
+ log_data["api_key"] = "api_key: " + std::to_string(params.api_keys.size()) + " keys loaded";
}
+ // Necessary similarity of prompt for slot selection
+ ctx_server.slot_prompt_similarity = params.slot_prompt_similarity;
+
// load the model
if (!ctx_server.load_model(params)) {
state.store(SERVER_STATE_ERROR);
@@ -3042,10 +2585,10 @@ int main(int argc, char ** argv) {
const auto model_meta = ctx_server.model_meta();
// if a custom chat template is not supplied, we will use the one that comes with the model (if any)
- if (sparams.chat_template.empty()) {
+ if (params.chat_template.empty()) {
if (!ctx_server.validate_model_chat_template()) {
LOG_ERROR("The chat template that comes with this model is not yet supported, falling back to chatml. This may cause the model to output suboptimal responses", {});
- sparams.chat_template = "chatml";
+ params.chat_template = "chatml";
}
}
@@ -3057,11 +2600,11 @@ int main(int argc, char ** argv) {
chat.push_back({{"role", "assistant"}, {"content", "Hi there"}});
chat.push_back({{"role", "user"}, {"content", "How are you?"}});
- const std::string chat_example = format_chat(ctx_server.model, sparams.chat_template, chat);
+ const std::string chat_example = format_chat(ctx_server.model, params.chat_template, chat);
LOG_INFO("chat template", {
{"chat_example", chat_example},
- {"built_in", sparams.chat_template.empty()},
+ {"built_in", params.chat_template.empty()},
});
}
@@ -3069,7 +2612,7 @@ int main(int argc, char ** argv) {
// Middlewares
//
- auto middleware_validate_api_key = [&sparams, &res_error](const httplib::Request & req, httplib::Response & res) {
+ auto middleware_validate_api_key = [¶ms, &res_error](const httplib::Request & req, httplib::Response & res) {
// TODO: should we apply API key to all endpoints, including "/health" and "/models"?
static const std::set protected_endpoints = {
"/props",
@@ -3087,7 +2630,7 @@ int main(int argc, char ** argv) {
};
// If API key is not set, skip validation
- if (sparams.api_keys.empty()) {
+ if (params.api_keys.empty()) {
return true;
}
@@ -3102,7 +2645,7 @@ int main(int argc, char ** argv) {
std::string prefix = "Bearer ";
if (auth_header.substr(0, prefix.size()) == prefix) {
std::string received_api_key = auth_header.substr(prefix.size());
- if (std::find(sparams.api_keys.begin(), sparams.api_keys.end(), received_api_key) != sparams.api_keys.end()) {
+ if (std::find(params.api_keys.begin(), params.api_keys.end(), received_api_key) != params.api_keys.end()) {
return true; // API key is valid
}
}
@@ -3157,7 +2700,7 @@ int main(int argc, char ** argv) {
};
res.status = 200; // HTTP OK
- if (sparams.slots_endpoint && req.has_param("include_slots")) {
+ if (params.endpoint_slots && req.has_param("include_slots")) {
health["slots"] = result.data.at("slots");
}
@@ -3183,7 +2726,7 @@ int main(int argc, char ** argv) {
};
const auto handle_slots = [&](const httplib::Request &, httplib::Response & res) {
- if (!sparams.slots_endpoint) {
+ if (!params.endpoint_slots) {
res_error(res, format_error_response("This server does not support slots endpoint.", ERROR_TYPE_NOT_SUPPORTED));
return;
}
@@ -3207,7 +2750,7 @@ int main(int argc, char ** argv) {
};
const auto handle_metrics = [&](const httplib::Request &, httplib::Response & res) {
- if (!sparams.metrics_endpoint) {
+ if (!params.endpoint_metrics) {
res_error(res, format_error_response("This server does not support metrics endpoint.", ERROR_TYPE_NOT_SUPPORTED));
return;
}
@@ -3307,14 +2850,14 @@ int main(int argc, char ** argv) {
res.status = 200; // HTTP OK
};
- const auto handle_slots_save = [&ctx_server, &res_error, &sparams](const httplib::Request & req, httplib::Response & res, int id_slot) {
+ const auto handle_slots_save = [&ctx_server, &res_error, ¶ms](const httplib::Request & req, httplib::Response & res, int id_slot) {
json request_data = json::parse(req.body);
std::string filename = request_data.at("filename");
if (!fs_validate_filename(filename)) {
res_error(res, format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST));
return;
}
- std::string filepath = sparams.slot_save_path + filename;
+ std::string filepath = params.slot_save_path + filename;
server_task task;
task.type = SERVER_TASK_TYPE_SLOT_SAVE;
@@ -3337,14 +2880,14 @@ int main(int argc, char ** argv) {
}
};
- const auto handle_slots_restore = [&ctx_server, &res_error, &sparams](const httplib::Request & req, httplib::Response & res, int id_slot) {
+ const auto handle_slots_restore = [&ctx_server, &res_error, ¶ms](const httplib::Request & req, httplib::Response & res, int id_slot) {
json request_data = json::parse(req.body);
std::string filename = request_data.at("filename");
if (!fs_validate_filename(filename)) {
res_error(res, format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST));
return;
}
- std::string filepath = sparams.slot_save_path + filename;
+ std::string filepath = params.slot_save_path + filename;
server_task task;
task.type = SERVER_TASK_TYPE_SLOT_RESTORE;
@@ -3519,9 +3062,9 @@ int main(int argc, char ** argv) {
res.set_content(models.dump(), "application/json; charset=utf-8");
};
- const auto handle_chat_completions = [&ctx_server, &sparams, &res_error](const httplib::Request & req, httplib::Response & res) {
+ const auto handle_chat_completions = [&ctx_server, ¶ms, &res_error](const httplib::Request & req, httplib::Response & res) {
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
- json data = oaicompat_completion_params_parse(ctx_server.model, json::parse(req.body), sparams.chat_template);
+ json data = oaicompat_completion_params_parse(ctx_server.model, json::parse(req.body), params.chat_template);
const int id_task = ctx_server.queue_tasks.get_new_id();
@@ -3746,17 +3289,29 @@ int main(int argc, char ** argv) {
//
// register static assets routes
- if (!sparams.public_path.empty()) {
+ if (!params.public_path.empty()) {
// Set the base directory for serving static files
- svr->set_base_dir(sparams.public_path);
+ svr->set_base_dir(params.public_path);
}
// using embedded static files
- svr->Get("/", handle_static_file(index_html, index_html_len, "text/html; charset=utf-8"));
- svr->Get("/index.js", handle_static_file(index_js, index_js_len, "text/javascript; charset=utf-8"));
- svr->Get("/completion.js", handle_static_file(completion_js, completion_js_len, "text/javascript; charset=utf-8"));
- svr->Get("/json-schema-to-grammar.mjs", handle_static_file(
- json_schema_to_grammar_mjs, json_schema_to_grammar_mjs_len, "text/javascript; charset=utf-8"));
+ svr->Get("/", handle_static_file(index_html, index_html_len, "text/html; charset=utf-8"));
+ svr->Get("/index.js", handle_static_file(index_js, index_js_len, "text/javascript; charset=utf-8"));
+ svr->Get("/completion.js", handle_static_file(completion_js, completion_js_len, "text/javascript; charset=utf-8"));
+ svr->Get("/json-schema-to-grammar.mjs", handle_static_file(json_schema_to_grammar_mjs, json_schema_to_grammar_mjs_len, "text/javascript; charset=utf-8"));
+
+ // add new-ui files
+ svr->Get("/colorthemes.css", handle_static_file(colorthemes_css, colorthemes_css_len, "text/css; charset=utf-8"));
+ svr->Get("/style.css", handle_static_file(style_css, style_css_len, "text/css; charset=utf-8"));
+ svr->Get("/theme-beeninorder.css", handle_static_file(theme_beeninorder_css, theme_beeninorder_css_len, "text/css; charset=utf-8"));
+ svr->Get("/theme-ketivah.css", handle_static_file(theme_ketivah_css, theme_ketivah_css_len, "text/css; charset=utf-8"));
+ svr->Get("/theme-mangotango.css", handle_static_file(theme_mangotango_css, theme_mangotango_css_len, "text/css; charset=utf-8"));
+ svr->Get("/theme-playground.css", handle_static_file(theme_playground_css, theme_playground_css_len, "text/css; charset=utf-8"));
+ svr->Get("/theme-polarnight.css", handle_static_file(theme_polarnight_css, theme_polarnight_css_len, "text/css; charset=utf-8"));
+ svr->Get("/theme-snowstorm.css", handle_static_file(theme_snowstorm_css, theme_snowstorm_css_len, "text/css; charset=utf-8"));
+ svr->Get("/index-new.html", handle_static_file(index_new_html, index_new_html_len, "text/html; charset=utf-8"));
+ svr->Get("/system-prompts.js", handle_static_file(system_prompts_js, system_prompts_js_len, "text/javascript; charset=utf-8"));
+ svr->Get("/prompt-formats.js", handle_static_file(prompt_formats_js, prompt_formats_js_len, "text/javascript; charset=utf-8"));
// register API routes
svr->Get ("/health", handle_health);
@@ -3775,7 +3330,7 @@ int main(int argc, char ** argv) {
svr->Post("/v1/embeddings", handle_embeddings);
svr->Post("/tokenize", handle_tokenize);
svr->Post("/detokenize", handle_detokenize);
- if (!sparams.slot_save_path.empty()) {
+ if (!params.slot_save_path.empty()) {
// only enable slot endpoints if slot_save_path is set
svr->Post("/slots/:id_slot", handle_slots_action);
}
@@ -3783,12 +3338,12 @@ int main(int argc, char ** argv) {
//
// Start the server
//
- if (sparams.n_threads_http < 1) {
+ if (params.n_threads_http < 1) {
// +2 threads for monitoring endpoints
- sparams.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1);
+ params.n_threads_http = std::max(params.n_parallel + 2, (int32_t) std::thread::hardware_concurrency() - 1);
}
- log_data["n_threads_http"] = std::to_string(sparams.n_threads_http);
- svr->new_task_queue = [&sparams] { return new httplib::ThreadPool(sparams.n_threads_http); };
+ log_data["n_threads_http"] = std::to_string(params.n_threads_http);
+ svr->new_task_queue = [¶ms] { return new httplib::ThreadPool(params.n_threads_http); };
LOG_INFO("HTTP server listening", log_data);
diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp
index d8a2286e4..63fde9c9f 100644
--- a/examples/server/utils.hpp
+++ b/examples/server/utils.hpp
@@ -116,13 +116,6 @@ static inline void server_log(const char * level, const char * function, int lin
// chat template utils
//
-// Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid
-inline bool verify_custom_template(const std::string & tmpl) {
- llama_chat_message chat[] = {{"user", "test"}};
- int res = llama_chat_apply_template(nullptr, tmpl.c_str(), chat, 1, true, nullptr, 0);
- return res >= 0;
-}
-
// Format given chat. If tmpl is empty, we take the template from model metadata
inline std::string format_chat(const struct llama_model * model, const std::string & tmpl, const std::vector & messages) {
size_t alloc_size = 0;
@@ -260,6 +253,13 @@ static size_t common_part(const std::vector & a, const std::vector<
return i;
}
+static size_t common_part(const std::string & a, const std::string & b) {
+ size_t i;
+ for (i = 0; i < a.size() && i < b.size() && a[i] == b[i]; i++) {}
+
+ return i;
+}
+
static bool ends_with(const std::string & str, const std::string & suffix) {
return str.size() >= suffix.size() && 0 == str.compare(str.size() - suffix.size(), suffix.size(), suffix);
}
diff --git a/examples/simple/README.md b/examples/simple/README.md
index 5d24b1046..49e24501c 100644
--- a/examples/simple/README.md
+++ b/examples/simple/README.md
@@ -3,7 +3,7 @@
The purpose of this example is to demonstrate a minimal usage of llama.cpp for generating text with a given prompt.
```bash
-./simple ./models/llama-7b-v2/ggml-model-f16.gguf "Hello my name is"
+./simple -m ./models/llama-7b-v2/ggml-model-f16.gguf -p "Hello my name is"
...
diff --git a/examples/simple/simple.cpp b/examples/simple/simple.cpp
index b0f8e0fdc..69a92cf7d 100644
--- a/examples/simple/simple.cpp
+++ b/examples/simple/simple.cpp
@@ -6,28 +6,27 @@
#include
#include
+static void print_usage(int argc, char ** argv, const gpt_params & params) {
+ gpt_params_print_usage(argc, argv, params);
+
+ LOG_TEE("\nexample usage:\n");
+ LOG_TEE("\n %s -m model.gguf -p \"Hello my name is\" -n 32\n", argv[0]);
+ LOG_TEE("\n");
+}
+
int main(int argc, char ** argv) {
gpt_params params;
- if (argc == 1 || argv[1][0] == '-') {
- printf("usage: %s MODEL_PATH [PROMPT]\n" , argv[0]);
- return 1 ;
- }
+ params.prompt = "Hello my name is";
+ params.n_predict = 32;
- if (argc >= 2) {
- params.model = argv[1];
- }
-
- if (argc >= 3) {
- params.prompt = argv[2];
- }
-
- if (params.prompt.empty()) {
- params.prompt = "Hello my name is";
+ if (!gpt_params_parse(argc, argv, params)) {
+ print_usage(argc, argv, params);
+ return 1;
}
// total length of the sequence including the prompt
- const int n_len = 32;
+ const int n_predict = params.n_predict;
// init LLM
@@ -36,9 +35,7 @@ int main(int argc, char ** argv) {
// initialize the model
- llama_model_params model_params = llama_model_default_params();
-
- // model_params.n_gpu_layers = 99; // offload all layers to the GPU
+ llama_model_params model_params = llama_model_params_from_gpt_params(params);
llama_model * model = llama_load_model_from_file(params.model.c_str(), model_params);
@@ -49,12 +46,7 @@ int main(int argc, char ** argv) {
// initialize the context
- llama_context_params ctx_params = llama_context_default_params();
-
- ctx_params.seed = 1234;
- ctx_params.n_ctx = 2048;
- ctx_params.n_threads = params.n_threads;
- ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch;
+ llama_context_params ctx_params = llama_context_params_from_gpt_params(params);
llama_context * ctx = llama_new_context_with_model(model, ctx_params);
@@ -69,14 +61,14 @@ int main(int argc, char ** argv) {
tokens_list = ::llama_tokenize(ctx, params.prompt, true);
const int n_ctx = llama_n_ctx(ctx);
- const int n_kv_req = tokens_list.size() + (n_len - tokens_list.size());
+ const int n_kv_req = tokens_list.size() + (n_predict - tokens_list.size());
- LOG_TEE("\n%s: n_len = %d, n_ctx = %d, n_kv_req = %d\n", __func__, n_len, n_ctx, n_kv_req);
+ LOG_TEE("\n%s: n_predict = %d, n_ctx = %d, n_kv_req = %d\n", __func__, n_predict, n_ctx, n_kv_req);
// make sure the KV cache is big enough to hold all the prompt and generated tokens
if (n_kv_req > n_ctx) {
LOG_TEE("%s: error: n_kv_req > n_ctx, the required KV cache size is not big enough\n", __func__);
- LOG_TEE("%s: either reduce n_len or increase n_ctx\n", __func__);
+ LOG_TEE("%s: either reduce n_predict or increase n_ctx\n", __func__);
return 1;
}
@@ -115,7 +107,7 @@ int main(int argc, char ** argv) {
const auto t_main_start = ggml_time_us();
- while (n_cur <= n_len) {
+ while (n_cur <= n_predict) {
// sample the next token
{
auto n_vocab = llama_n_vocab(model);
@@ -134,7 +126,7 @@ int main(int argc, char ** argv) {
const llama_token new_token_id = llama_sample_token_greedy(ctx, &candidates_p);
// is it an end of generation?
- if (llama_token_is_eog(model, new_token_id) || n_cur == n_len) {
+ if (llama_token_is_eog(model, new_token_id) || n_cur == n_predict) {
LOG_TEE("\n");
break;
diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp
index 12e46fbc9..0939a1a6a 100644
--- a/examples/speculative/speculative.cpp
+++ b/examples/speculative/speculative.cpp
@@ -27,7 +27,8 @@ struct seq_draft {
int main(int argc, char ** argv) {
gpt_params params;
- if (gpt_params_parse(argc, argv, params) == false) {
+ if (!gpt_params_parse(argc, argv, params)) {
+ gpt_params_print_usage(argc, argv, params);
return 1;
}
diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp
index e2f85c682..b779f6bd4 100644
--- a/examples/train-text-from-scratch/train-text-from-scratch.cpp
+++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp
@@ -302,7 +302,7 @@ static struct ggml_tensor * llama_build_train_graphs(
const int rope_mode = 0;
return ggml_rope_ext(
- ctx, t, KQ_pos, nullptr, n_rot, rope_mode, n_ctx, 0, rope_freq_base, rope_freq_scale, 0.0f, 1.0f, 0.0f, 0.0f
+ ctx, t, KQ_pos, nullptr, n_rot, rope_mode, n_ctx, rope_freq_base, rope_freq_scale, 0.0f, 1.0f, 0.0f, 0.0f
);
};
diff --git a/flake.lock b/flake.lock
index fd6e2a5f6..09047ab10 100644
--- a/flake.lock
+++ b/flake.lock
@@ -5,11 +5,11 @@
"nixpkgs-lib": "nixpkgs-lib"
},
"locked": {
- "lastModified": 1715865404,
- "narHash": "sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78=",
+ "lastModified": 1717285511,
+ "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
"owner": "hercules-ci",
"repo": "flake-parts",
- "rev": "8dc45382d5206bd292f9c2768b8058a8fd8311d9",
+ "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
"type": "github"
},
"original": {
@@ -20,11 +20,11 @@
},
"nixpkgs": {
"locked": {
- "lastModified": 1716509168,
- "narHash": "sha256-4zSIhSRRIoEBwjbPm3YiGtbd8HDWzFxJjw5DYSDy1n8=",
+ "lastModified": 1716948383,
+ "narHash": "sha256-SzDKxseEcHR5KzPXLwsemyTR/kaM9whxeiJohbL04rs=",
"owner": "NixOS",
"repo": "nixpkgs",
- "rev": "bfb7a882678e518398ce9a31a881538679f6f092",
+ "rev": "ad57eef4ef0659193044870c731987a6df5cf56b",
"type": "github"
},
"original": {
@@ -36,14 +36,14 @@
},
"nixpkgs-lib": {
"locked": {
- "lastModified": 1714640452,
- "narHash": "sha256-QBx10+k6JWz6u7VsohfSw8g8hjdBZEf8CFzXH1/1Z94=",
+ "lastModified": 1717284937,
+ "narHash": "sha256-lIbdfCsf8LMFloheeE6N31+BMIeixqyQWbSr2vk79EQ=",
"type": "tarball",
- "url": "https://github.com/NixOS/nixpkgs/archive/50eb7ecf4cd0a5756d7275c8ba36790e5bd53e33.tar.gz"
+ "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz"
},
"original": {
"type": "tarball",
- "url": "https://github.com/NixOS/nixpkgs/archive/50eb7ecf4cd0a5756d7275c8ba36790e5bd53e33.tar.gz"
+ "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz"
}
},
"root": {
diff --git a/flake.nix b/flake.nix
index 9cd3756e5..0a52ea52e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -159,7 +159,6 @@
windows = config.legacyPackages.llamaPackagesWindows.llama-cpp;
}
// lib.optionalAttrs pkgs.stdenv.isLinux {
- opencl = config.packages.default.override { useOpenCL = true; };
cuda = config.legacyPackages.llamaPackagesCuda.llama-cpp;
mpi-cpu = config.packages.default.override { useMpi = true; };
diff --git a/ggml-alloc.c b/ggml-alloc.c
index 1fbd376ed..73a3c1575 100644
--- a/ggml-alloc.c
+++ b/ggml-alloc.c
@@ -377,7 +377,7 @@ ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs
galloc->bufts = calloc(n_bufs, sizeof(ggml_backend_buffer_type_t));
GGML_ASSERT(galloc->bufts != NULL);
- galloc->buffers = calloc(n_bufs, sizeof(ggml_backend_buffer_t) * n_bufs);
+ galloc->buffers = calloc(n_bufs, sizeof(ggml_backend_buffer_t));
GGML_ASSERT(galloc->buffers != NULL);
galloc->buf_tallocs = calloc(n_bufs, sizeof(struct ggml_dyn_tallocr *));
@@ -750,7 +750,7 @@ static void ggml_gallocr_init_tensor(ggml_gallocr_t galloc, struct ggml_tensor *
// this tensor was allocated without ggml-backend
return;
}
- ggml_backend_view_init(galloc->buffers[buffer_id], tensor);
+ ggml_backend_view_init(tensor);
}
} else {
if (tensor->data == NULL) {
@@ -899,12 +899,12 @@ static bool alloc_tensor_range(struct ggml_context * ctx,
if (t->view_src == NULL) {
ggml_tallocr_alloc(&tallocr, t);
} else if (t->buffer == NULL) {
- ggml_backend_view_init(buffer, t);
+ ggml_backend_view_init(t);
}
} else {
if (t->view_src != NULL && t->buffer == NULL) {
// view of a pre-allocated tensor
- ggml_backend_view_init(buffer, t);
+ ggml_backend_view_init(t);
}
}
}
diff --git a/ggml-backend.c b/ggml-backend.c
index 9e35ce98d..05737ed69 100644
--- a/ggml-backend.c
+++ b/ggml-backend.c
@@ -151,7 +151,7 @@ void ggml_backend_buffer_reset(ggml_backend_buffer_t buffer) {
bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst) {
ggml_backend_buffer_t dst_buf = dst->view_src ? dst->view_src->buffer : dst->buffer;
if (dst_buf->iface.cpy_tensor) {
- return src->buffer->iface.cpy_tensor(dst_buf, src, dst);
+ return dst_buf->iface.cpy_tensor(dst_buf, src, dst);
}
return false;
}
@@ -1887,15 +1887,15 @@ ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched,
// utils
-void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {
+void ggml_backend_view_init(struct ggml_tensor * tensor) {
GGML_ASSERT(tensor->buffer == NULL);
GGML_ASSERT(tensor->view_src != NULL);
GGML_ASSERT(tensor->view_src->buffer != NULL);
GGML_ASSERT(tensor->view_src->data != NULL);
- tensor->buffer = buffer;
+ tensor->buffer = tensor->view_src->buffer;
tensor->data = (char *)tensor->view_src->data + tensor->view_offs;
- ggml_backend_buffer_init_tensor(buffer, tensor);
+ ggml_backend_buffer_init_tensor(tensor->buffer, tensor);
}
void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr) {
@@ -1954,7 +1954,7 @@ static void graph_copy_init_tensor(struct ggml_hash_set hash_set, struct ggml_te
struct ggml_tensor * dst = node_copies[id];
if (dst->view_src != NULL) {
graph_copy_init_tensor(hash_set, node_copies, node_init, src->view_src);
- ggml_backend_view_init(dst->view_src->buffer, dst);
+ ggml_backend_view_init(dst);
}
else {
ggml_backend_tensor_copy(src, dst);
diff --git a/ggml-backend.h b/ggml-backend.h
index 744b6a774..c582b0685 100644
--- a/ggml-backend.h
+++ b/ggml-backend.h
@@ -225,7 +225,7 @@ extern "C" {
// Tensor initialization
GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
- GGML_API void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
+ GGML_API void ggml_backend_view_init(struct ggml_tensor * tensor);
#ifdef __cplusplus
diff --git a/ggml-common.h b/ggml-common.h
index 77e6bfba4..e8efceb76 100644
--- a/ggml-common.h
+++ b/ggml-common.h
@@ -123,12 +123,18 @@ typedef sycl::half2 ggml_half2;
#define QI1_S (QK_K / (4*QR1_S))
#define QR1_S 8
+#define QI1_M (QK_K / (4*QR1_M))
+#define QR1_M 8
+
#define QI4_NL (QK4_NL / (4*QR4_NL))
#define QR4_NL 2
#define QI4_XS (QK_K / (4*QR4_XS))
#define QR4_XS 8
+#define QI3_S (QK_K / (4*QR3_S))
+#define QR3_S 8
+
#endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP
#define QK4_0 32
diff --git a/ggml-cuda.cu b/ggml-cuda.cu
index daaa0cd6a..dad8a9e2d 100644
--- a/ggml-cuda.cu
+++ b/ggml-cuda.cu
@@ -633,88 +633,22 @@ GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {
// cuda split buffer
-static int64_t get_row_rounding(ggml_type type, const std::array & tensor_split) {
- int64_t min_compute_capability = INT_MAX;
- int64_t max_compute_capability = INT_MIN;
+static int64_t get_row_rounding(const std::array & tensor_split) {
+ int64_t row_rounding = 0;
for (int id = 0; id < ggml_backend_cuda_get_device_count(); ++id) {
- if (tensor_split[id] < (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {
- if (min_compute_capability > ggml_cuda_info().devices[id].cc) {
- min_compute_capability = ggml_cuda_info().devices[id].cc;
- }
- if (max_compute_capability < ggml_cuda_info().devices[id].cc) {
- max_compute_capability = ggml_cuda_info().devices[id].cc;
- }
+ if (tensor_split[id] >= (id + 1 < ggml_backend_cuda_get_device_count() ? tensor_split[id + 1] : 1.0f)) {
+ continue;
}
- }
-#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
- switch(type) {
- case GGML_TYPE_Q4_0:
- case GGML_TYPE_Q4_1:
- case GGML_TYPE_Q5_0:
- case GGML_TYPE_Q5_1:
- case GGML_TYPE_Q8_0:
- return max_compute_capability >= CC_RDNA2 ? 128 : 64;
- case GGML_TYPE_F16:
- case GGML_TYPE_F32:
- return 1;
- case GGML_TYPE_Q2_K:
- return max_compute_capability >= CC_RDNA2 ? 128 : 32;
- case GGML_TYPE_Q3_K:
- return min_compute_capability < CC_RDNA2 ? 128 : 64;
- case GGML_TYPE_Q4_K:
- case GGML_TYPE_Q5_K:
- case GGML_TYPE_Q6_K:
- case GGML_TYPE_IQ2_XXS:
- case GGML_TYPE_IQ2_XS:
- case GGML_TYPE_IQ2_S:
- case GGML_TYPE_IQ3_XXS:
- case GGML_TYPE_IQ1_S:
- case GGML_TYPE_IQ1_M:
- case GGML_TYPE_IQ4_NL:
- case GGML_TYPE_IQ4_XS:
- case GGML_TYPE_IQ3_S:
- return max_compute_capability >= CC_RDNA2 ? 128 : 64;
- default:
- GGML_ASSERT(false);
+ const int cc = ggml_cuda_info().devices[id].cc;
+ row_rounding = std::max(row_rounding, (int64_t)get_mmq_y_host(cc, get_mmq_x_max_host(cc)));
}
-#else
- switch(type) {
- case GGML_TYPE_Q4_0:
- case GGML_TYPE_Q4_1:
- return max_compute_capability >= CC_VOLTA ? 128 : 64;
- case GGML_TYPE_Q5_0:
- case GGML_TYPE_Q5_1:
- case GGML_TYPE_Q8_0:
- return 64;
- case GGML_TYPE_F16:
- case GGML_TYPE_F32:
- return 1;
- case GGML_TYPE_Q2_K:
- case GGML_TYPE_Q3_K:
- case GGML_TYPE_Q4_K:
- case GGML_TYPE_Q5_K:
- case GGML_TYPE_IQ2_XXS:
- case GGML_TYPE_IQ2_XS:
- case GGML_TYPE_IQ2_S:
- case GGML_TYPE_IQ3_XXS:
- case GGML_TYPE_IQ1_S:
- case GGML_TYPE_IQ1_M:
- case GGML_TYPE_IQ4_NL:
- case GGML_TYPE_IQ4_XS:
- case GGML_TYPE_IQ3_S:
- return max_compute_capability >= CC_VOLTA ? 128 : 64;
- case GGML_TYPE_Q6_K:
- return 64;
- default:
- GGML_ASSERT(false);
- }
-#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)
+ return row_rounding;
}
static void get_row_split(int64_t * row_low, int64_t * row_high, const ggml_tensor * tensor, const std::array & tensor_split, int id) {
const int64_t nrows = ggml_nrows(tensor);
- const int64_t rounding = get_row_rounding(tensor->type, tensor_split);
+ const int64_t rounding = get_row_rounding(tensor_split);
*row_low = id == 0 ? 0 : nrows*tensor_split[id];
*row_low -= *row_low % rounding;
@@ -1499,7 +1433,7 @@ static void ggml_cuda_op_mul_mat(
// for multi GPU, get the row boundaries from tensor split
// and round to mul_mat_q tile sizes
if (split) {
- const int64_t rounding = get_row_rounding(src0->type, tensor_split);
+ const int64_t rounding = get_row_rounding(tensor_split);
if (id != 0) {
dev[id].row_low = ne01*tensor_split[id];
@@ -2702,10 +2636,8 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t
if (cuda_graph_update_required) {
// Extract nodes from graph
- if (cuda_ctx->cuda_graph->num_nodes == 0) {
- // First call with null argument gets number of nodes in graph
- CUDA_CHECK(cudaGraphGetNodes(cuda_ctx->cuda_graph->graph, nullptr, &cuda_ctx->cuda_graph->num_nodes));
- }
+ // First call with null argument gets number of nodes in graph
+ CUDA_CHECK(cudaGraphGetNodes(cuda_ctx->cuda_graph->graph, nullptr, &cuda_ctx->cuda_graph->num_nodes));
// Subsequent call with non-null argument gets nodes
cuda_ctx->cuda_graph->nodes.resize(cuda_ctx->cuda_graph->num_nodes);
cuda_ctx->cuda_graph->params.resize(cuda_ctx->cuda_graph->num_nodes);
diff --git a/ggml-cuda/common.cuh b/ggml-cuda/common.cuh
index 22872ca5c..90a0a81ea 100644
--- a/ggml-cuda/common.cuh
+++ b/ggml-cuda/common.cuh
@@ -160,7 +160,7 @@
#endif
#define MMVQ_MAX_BATCH_SIZE 8 // max batch size to use MMVQ kernels
-#define MMQ_MAX_BATCH_SIZE 32 // max batch size to use MMQ kernels when tensor cores are available
+#define MMQ_MAX_BATCH_SIZE 64 // max batch size to use MMQ kernels when tensor cores are available
#define MATRIX_ROW_PADDING 512 // last row of quant. matrices is a multiple of this to avoid out-of-bounds memory accesses
@@ -484,6 +484,161 @@ static __device__ __forceinline__ float get_alibi_slope(
return powf(base, exph);
}
+template
+struct ggml_cuda_type_traits;
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = 1;
+ static constexpr int qr = 1;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK4_0;
+ static constexpr int qr = QR4_0;
+ static constexpr int qi = QI4_0;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK4_1;
+ static constexpr int qr = QR4_1;
+ static constexpr int qi = QI4_1;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK5_0;
+ static constexpr int qr = QR5_0;
+ static constexpr int qi = QI5_0;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK5_1;
+ static constexpr int qr = QR5_1;
+ static constexpr int qi = QI5_1;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK8_0;
+ static constexpr int qr = QR8_0;
+ static constexpr int qi = QI8_0;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_K;
+ static constexpr int qi = QI2_K;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR3_K;
+ static constexpr int qi = QI3_K;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR4_K;
+ static constexpr int qi = QI4_K;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR5_K;
+ static constexpr int qi = QI5_K;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR6_K;
+ static constexpr int qi = QI6_K;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_XXS;
+ static constexpr int qi = QI2_XXS;
+};
+
+template<>
+struct ggml_cuda_type_traits {
+ static constexpr int qk = QK_K;
+ static constexpr int qr = QR2_XS;
+ static constexpr int qi = QI2_XS;
+};
+
+template<>
+struct ggml_cuda_type_traits
SimpleChat
-
-
-
-
+
-
+
@@ -40,7 +42,7 @@
-
+
diff --git a/examples/server/public_simplechat/readme.md b/examples/server/public_simplechat/readme.md
index de0dfc99d..36a46885d 100644
--- a/examples/server/public_simplechat/readme.md
+++ b/examples/server/public_simplechat/readme.md
@@ -11,18 +11,29 @@ in a simple way with minimal code from a common code base. Inturn additionally i
multiple independent back and forth chatting to an extent, with the ai llm model at a basic level, with their
own system prompts.
+This allows seeing the generated text / ai-model response in oneshot at the end, after it is fully generated,
+or potentially as it is being generated, in a streamed manner from the server/ai-model.
+
+Auto saves the chat session locally as and when the chat is progressing and inturn at a later time when you
+open SimpleChat, option is provided to restore the old chat session, if a matching one exists.
+
The UI follows a responsive web design so that the layout can adapt to available display space in a usable
enough manner, in general.
Allows developer/end-user to control some of the behaviour by updating gMe members from browser's devel-tool
-console.
+console. Parallely some of the directly useful to end-user settings can also be changed using the provided
+settings ui.
-NOTE: Given that the idea is for basic minimal testing, it doesnt bother with any model context length and
-culling of old messages from the chat by default. However by enabling the sliding window chat logic, a crude
-form of old messages culling can be achieved.
+NOTE: Current web service api doesnt expose the model context length directly, so client logic doesnt provide
+any adaptive culling of old messages nor of replacing them with summary of their content etal. However there
+is a optional sliding window based chat logic, which provides a simple minded culling of old messages from
+the chat history before sending to the ai model.
-NOTE: It doesnt set any parameters other than temperature and max_tokens for now. However if someone wants
-they can update the js file or equivalent member in gMe as needed.
+NOTE: Wrt options sent with the request, it mainly sets temperature, max_tokens and optionaly stream for now.
+However if someone wants they can update the js file or equivalent member in gMe as needed.
+
+NOTE: One may be able to use this to chat with openai api web-service /chat/completions endpoint, in a very
+limited / minimal way. One will need to set model, openai url and authorization bearer key in settings ui.
## usage
@@ -52,9 +63,15 @@ Open this simple web front end from your local browser
Once inside
-* Select between chat and completion mode. By default it is set to chat mode.
+* If you want to, you can change many of the default global settings
+ * the base url (ie ip addr / domain name, port)
+ * chat (default) vs completion mode
+ * try trim garbage in response or not
+ * amount of chat history in the context sent to server/ai-model
+ * oneshot or streamed mode.
* In completion mode
+ * one normally doesnt use a system prompt in completion mode.
* logic by default doesnt insert any role specific "ROLE: " prefix wrt each role's message.
If the model requires any prefix wrt user role messages, then the end user has to
explicitly add the needed prefix, when they enter their chat message.
@@ -88,12 +105,16 @@ Once inside
* Wait for the logic to communicate with the server and get the response.
* the user is not allowed to enter any fresh query during this time.
* the user input box will be disabled and a working message will be shown in it.
+ * if trim garbage is enabled, the logic will try to trim repeating text kind of garbage to some extent.
* just refresh the page, to reset wrt the chat history and or system prompt and start afresh.
* Using NewChat one can start independent chat sessions.
* two independent chat sessions are setup by default.
+* When you want to print, switching ChatHistoryInCtxt to Full and clicking on the chat session button of
+ interest, will display the full chat history till then wrt same, if you want full history for printing.
+
## Devel note
@@ -104,14 +125,31 @@ by developers who may not be from web frontend background (so inturn may not be
end-use-specific-language-extensions driven flows) so that they can use it to explore/experiment things.
And given that the idea is also to help explore/experiment for developers, some flexibility is provided
-to change behaviour easily using the devel-tools/console, for now. And skeletal logic has been implemented
-to explore some of the end points and ideas/implications around them.
+to change behaviour easily using the devel-tools/console or provided minimal settings ui (wrt few aspects).
+Skeletal logic has been implemented to explore some of the end points and ideas/implications around them.
### General
Me/gMe consolidates the settings which control the behaviour into one object.
One can see the current settings, as well as change/update them using browsers devel-tool/console.
+It is attached to the document object. Some of these can also be updated using the Settings UI.
+
+ baseURL - the domain-name/ip-address and inturn the port to send the request.
+
+ bStream - control between oneshot-at-end and live-stream-as-its-generated collating and showing
+ of the generated response.
+
+ the logic assumes that the text sent from the server follows utf-8 encoding.
+
+ in streaming mode - if there is any exception, the logic traps the same and tries to ensure
+ that text generated till then is not lost.
+
+ if a very long text is being generated, which leads to no user interaction for sometime and
+ inturn the machine goes into power saving mode or so, the platform may stop network connection,
+ leading to exception.
+
+ apiEP - select between /completions and /chat/completions endpoint provided by the server/ai-model.
bCompletionFreshChatAlways - whether Completion mode collates complete/sliding-window history when
communicating with the server or only sends the latest user query/message.
@@ -119,6 +157,19 @@ One can see the current settings, as well as change/update them using browsers d
bCompletionInsertStandardRolePrefix - whether Completion mode inserts role related prefix wrt the
messages that get inserted into prompt field wrt /Completion endpoint.
+ bTrimGarbage - whether garbage repeatation at the end of the generated ai response, should be
+ trimmed or left as is. If enabled, it will be trimmed so that it wont be sent back as part of
+ subsequent chat history. At the same time the actual trimmed text is shown to the user, once
+ when it was generated, so user can check if any useful info/data was there in the response.
+
+ One may be able to request the ai-model to continue (wrt the last response) (if chat-history
+ is enabled as part of the chat-history-in-context setting), and chances are the ai-model will
+ continue starting from the trimmed part, thus allows long response to be recovered/continued
+ indirectly, in many cases.
+
+ The histogram/freq based trimming logic is currently tuned for english language wrt its
+ is-it-a-alpabetic|numeral-char regex match logic.
+
chatRequestOptions - maintains the list of options/fields to send along with chat request,
irrespective of whether /chat/completions or /completions endpoint.
@@ -126,6 +177,14 @@ One can see the current settings, as well as change/update them using browsers d
modify the existing options value or remove them, for now you can update this global var
using browser's development-tools/console.
+ For string and numeric fields in chatRequestOptions, including even those added by a user
+ at runtime by directly modifying gMe.chatRequestOptions, setting ui entries will be auto
+ created.
+
+ headers - maintains the list of http headers sent when request is made to the server. By default
+ Content-Type is set to application/json. Additionally Authorization entry is provided, which can
+ be set if needed using the settings ui.
+
iRecentUserMsgCnt - a simple minded SlidingWindow to limit context window load at Ai Model end.
This is disabled by default. However if enabled, then in addition to latest system message, only
the last/latest iRecentUserMsgCnt user messages after the latest system prompt and its responses
@@ -140,7 +199,8 @@ One can see the current settings, as well as change/update them using browsers d
By using gMe's iRecentUserMsgCnt and chatRequestOptions.max_tokens one can try to control the
implications of loading of the ai-model's context window by chat history, wrt chat response to
-some extent in a simple crude way.
+some extent in a simple crude way. You may also want to control the context size enabled when
+the server loads ai-model, on the server end.
Sometimes the browser may be stuborn with caching of the file, so your updates to html/css/js
@@ -149,28 +209,15 @@ matter clearing site data, dont directly override site caching in all cases. Wor
have to change port. Or in dev tools of browser, you may be able to disable caching fully.
-Concept of multiple chat sessions with different servers, as well as saving and restoring of
-those across browser usage sessions, can be woven around the SimpleChat/MultiChatUI class and
-its instances relatively easily, however given the current goal of keeping this simple, it has
-not been added, for now.
+Currently the server to communicate with is maintained globally and not as part of a specific
+chat session. So if one changes the server ip/url in setting, then all chat sessions will auto
+switch to this new server, when you try using those sessions.
By switching between chat.add_system_begin/anytime, one can control whether one can change
the system prompt, anytime during the conversation or only at the beginning.
-read_json_early, is to experiment with reading json response data early on, if available,
-so that user can be shown generated data, as and when it is being generated, rather than
-at the end when full data is available.
-
- the server flow doesnt seem to be sending back data early, atleast for request (inc options)
- that is currently sent.
-
- if able to read json data early on in future, as and when ai model is generating data, then
- this helper needs to indirectly update the chat div with the recieved data, without waiting
- for the overall data to be available.
-
-
### Default setup
By default things are setup to try and make the user experience a bit better, if possible.
@@ -179,7 +226,8 @@ However a developer when testing the server of ai-model may want to change these
Using iRecentUserMsgCnt reduce chat history context sent to the server/ai-model to be
just the system-prompt, prev-user-request-and-ai-response and cur-user-request, instead of
full chat history. This way if there is any response with garbage/repeatation, it doesnt
-mess with things beyond the next question/request/query, in some ways.
+mess with things beyond the next question/request/query, in some ways. The trim garbage
+option also tries to help avoid issues with garbage in the context to an extent.
Set max_tokens to 1024, so that a relatively large previous reponse doesnt eat up the space
available wrt next query-response. However dont forget that the server when started should
@@ -189,11 +237,33 @@ also be started with a model context size of 1k or more, to be on safe side.
internal n_predict, for now add the same here on the client side, maybe later add max_tokens
to /completions endpoint handling code on server side.
-Frequency and presence penalty fields are set to 1.2 in the set of fields sent to server
-along with the user query. So that the model is partly set to try avoid repeating text in
-its response.
+NOTE: One may want to experiment with frequency/presence penalty fields in chatRequestOptions
+wrt the set of fields sent to server along with the user query. To check how the model behaves
+wrt repeatations in general in the generated text response.
-A end-user can change these behaviour by editing gMe from browser's devel-tool/console.
+A end-user can change these behaviour by editing gMe from browser's devel-tool/console or by
+using the providing settings ui.
+
+
+### OpenAi / Equivalent API WebService
+
+One may be abe to handshake with OpenAI/Equivalent api web service's /chat/completions endpoint
+for a minimal chatting experimentation by setting the below.
+
+* the baseUrl in settings ui
+ * https://api.openai.com/v1 or similar
+
+* Wrt request body - gMe.chatRequestOptions
+ * model (settings ui)
+ * any additional fields if required in future
+
+* Wrt request headers - gMe.headers
+ * Authorization (available through settings ui)
+ * Bearer THE_OPENAI_API_KEY
+ * any additional optional header entries like "OpenAI-Organization", "OpenAI-Project" or so
+
+NOTE: Not tested, as there is no free tier api testing available. However logically this might
+work.
## At the end
diff --git a/examples/server/public_simplechat/simplechat.css b/examples/server/public_simplechat/simplechat.css
index 20c738b12..13bfb80b4 100644
--- a/examples/server/public_simplechat/simplechat.css
+++ b/examples/server/public_simplechat/simplechat.css
@@ -21,6 +21,17 @@
.role-user {
background-color: lightgray;
}
+.role-trim {
+ background-color: lightpink;
+}
+
+.gridx2 {
+ display: grid;
+ grid-template-columns: repeat(2, 1fr);
+ border-bottom-style: dotted;
+ border-bottom-width: thin;
+ border-bottom-color: lightblue;
+}
.flex-grow {
flex-grow: 1;
diff --git a/examples/server/public_simplechat/simplechat.js b/examples/server/public_simplechat/simplechat.js
index 0c48da879..25afb2564 100644
--- a/examples/server/public_simplechat/simplechat.js
+++ b/examples/server/public_simplechat/simplechat.js
@@ -2,6 +2,9 @@
// A simple completions and chat/completions test related web front end logic
// by Humans for All
+import * as du from "./datautils.mjs";
+import * as ui from "./ui.mjs"
+
class Roles {
static System = "system";
static User = "user";
@@ -9,40 +12,65 @@ class Roles {
}
class ApiEP {
- static Chat = "chat";
- static Completion = "completion";
+ static Type = {
+ Chat: "chat",
+ Completion: "completion",
+ }
+ static UrlSuffix = {
+ 'chat': `/chat/completions`,
+ 'completion': `/completions`,
+ }
+
+ /**
+ * Build the url from given baseUrl and apiEp id.
+ * @param {string} baseUrl
+ * @param {string} apiEP
+ */
+ static Url(baseUrl, apiEP) {
+ if (baseUrl.endsWith("/")) {
+ baseUrl = baseUrl.substring(0, baseUrl.length-1);
+ }
+ return `${baseUrl}${this.UrlSuffix[apiEP]}`;
+ }
+
}
+
let gUsageMsg = `
Usage
-
-
- Set system prompt above, to try control ai response charactersitic, if model supports same. +
- System prompt above, to try control ai response characteristics.
- Completion mode normally wont have a system prompt. +
- Completion mode - no system prompt normally.
- Use shift+enter for inserting enter/newline.
- Enter your query to ai assistant below. -
- Completion mode doesnt insert user/role: prefix implicitly. -
- Use shift+enter for inserting enter/newline. -
- Default ContextWindow = [System, Last Query+Resp, Cur Query].
- experiment iRecentUserMsgCnt, max_tokens, model ctxt window to expand +
- ChatHistInCtxt, MaxTokens, ModelCtxt window to expand
-
-
-
-
-
-
Restore
+Load previously saved chat session, if available
`; + let btn = ui.el_create_button(chat.ods_key(), (ev)=>{ + console.log("DBUG:SimpleChat:SC:Load", chat); + chat.load(); + queueMicrotask(()=>{ + chat.show(div); + this.multiChat.elInSystem.value = chat.get_system_latest(); + }); + }); + div.appendChild(btn); + } + + /** + * Show the configurable parameters info in the passed Div element. + * @param {HTMLDivElement} elDiv + * @param {boolean} bAll + */ + show_info(elDiv, bAll=false) { + + let p = ui.el_create_append_p("Settings (devel-tools-console document[gMe])", elDiv); + p.className = "role-system"; + + if (bAll) { + + ui.el_create_append_p(`baseURL:${this.baseURL}`, elDiv); + + ui.el_create_append_p(`Authorization:${this.headers["Authorization"]}`, elDiv); + + ui.el_create_append_p(`bStream:${this.bStream}`, elDiv); + + ui.el_create_append_p(`bCompletionFreshChatAlways:${this.bCompletionFreshChatAlways}`, elDiv); + + ui.el_create_append_p(`bCompletionInsertStandardRolePrefix:${this.bCompletionInsertStandardRolePrefix}`, elDiv); + + ui.el_create_append_p(`bTrimGarbage:${this.bTrimGarbage}`, elDiv); + + ui.el_create_append_p(`iRecentUserMsgCnt:${this.iRecentUserMsgCnt}`, elDiv); + + ui.el_create_append_p(`ApiEndPoint:${this.apiEP}`, elDiv); + + } + + ui.el_create_append_p(`chatRequestOptions:${JSON.stringify(this.chatRequestOptions, null, " - ")}`, elDiv); + ui.el_create_append_p(`headers:${JSON.stringify(this.headers, null, " - ")}`, elDiv); + + } + + /** + * Auto create ui input elements for fields in ChatRequestOptions + * Currently supports text and number field types. * @param {HTMLDivElement} elDiv */ - show_info(elDiv) { + show_settings_chatrequestoptions(elDiv) { + let typeDict = { + "string": "text", + "number": "number", + }; + let fs = document.createElement("fieldset"); + let legend = document.createElement("legend"); + legend.innerText = "ChatRequestOptions"; + fs.appendChild(legend); + elDiv.appendChild(fs); + for(const k in this.chatRequestOptions) { + let val = this.chatRequestOptions[k]; + let type = typeof(val); + if (!((type == "string") || (type == "number"))) { + continue; + } + let inp = ui.el_creatediv_input(`Set${k}`, k, typeDict[type], this.chatRequestOptions[k], (val)=>{ + if (type == "number") { + val = Number(val); + } + this.chatRequestOptions[k] = val; + }); + fs.appendChild(inp.div); + } + } - var p = document.createElement("p"); - p.innerText = "Settings (devel-tools-console gMe)"; - p.className = "role-system"; - elDiv.appendChild(p); + /** + * Show settings ui for configurable parameters, in the passed Div element. + * @param {HTMLDivElement} elDiv + */ + show_settings(elDiv) { - var p = document.createElement("p"); - p.innerText = `bCompletionFreshChatAlways:${this.bCompletionFreshChatAlways}`; - elDiv.appendChild(p); + let inp = ui.el_creatediv_input("SetBaseURL", "BaseURL", "text", this.baseURL, (val)=>{ + this.baseURL = val; + }); + elDiv.appendChild(inp.div); - p = document.createElement("p"); - p.innerText = `bCompletionInsertStandardRolePrefix:${this.bCompletionInsertStandardRolePrefix}`; - elDiv.appendChild(p); + inp = ui.el_creatediv_input("SetAuthorization", "Authorization", "text", this.headers["Authorization"], (val)=>{ + this.headers["Authorization"] = val; + }); + inp.el.placeholder = "Bearer OPENAI_API_KEY"; + elDiv.appendChild(inp.div); - p = document.createElement("p"); - p.innerText = `iRecentUserMsgCnt:${this.iRecentUserMsgCnt}`; - elDiv.appendChild(p); + let bb = ui.el_creatediv_boolbutton("SetStream", "Stream", {true: "[+] yes stream", false: "[-] do oneshot"}, this.bStream, (val)=>{ + this.bStream = val; + }); + elDiv.appendChild(bb.div); - p = document.createElement("p"); - p.innerText = `chatRequestOptions:${JSON.stringify(this.chatRequestOptions)}`; - elDiv.appendChild(p); + bb = ui.el_creatediv_boolbutton("SetCompletionFreshChatAlways", "CompletionFreshChatAlways", {true: "[+] yes fresh", false: "[-] no, with history"}, this.bCompletionFreshChatAlways, (val)=>{ + this.bCompletionFreshChatAlways = val; + }); + elDiv.appendChild(bb.div); + + bb = ui.el_creatediv_boolbutton("SetCompletionInsertStandardRolePrefix", "CompletionInsertStandardRolePrefix", {true: "[+] yes insert", false: "[-] dont insert"}, this.bCompletionInsertStandardRolePrefix, (val)=>{ + this.bCompletionInsertStandardRolePrefix = val; + }); + elDiv.appendChild(bb.div); + + bb = ui.el_creatediv_boolbutton("SetTrimGarbage", "TrimGarbage", {true: "[+] yes trim", false: "[-] dont trim"}, this.bTrimGarbage, (val)=>{ + this.bTrimGarbage = val; + }); + elDiv.appendChild(bb.div); + + let sel = ui.el_creatediv_select("SetChatHistoryInCtxt", "ChatHistoryInCtxt", this.sRecentUserMsgCnt, this.iRecentUserMsgCnt, (val)=>{ + this.iRecentUserMsgCnt = this.sRecentUserMsgCnt[val]; + }); + elDiv.appendChild(sel.div); + + sel = ui.el_creatediv_select("SetApiEP", "ApiEndPoint", ApiEP.Type, this.apiEP, (val)=>{ + this.apiEP = ApiEP.Type[val]; + }); + elDiv.appendChild(sel.div); + + this.show_settings_chatrequestoptions(elDiv); } @@ -619,6 +908,9 @@ let gMe; function startme() { console.log("INFO:SimpleChat:StartMe:Starting..."); gMe = new Me(); + gMe.debug_disable(); + document["gMe"] = gMe; + document["du"] = du; for (let cid of gMe.defaultChatIds) { gMe.multiChat.new_chat_session(cid); } diff --git a/examples/server/public_simplechat/ui.mjs b/examples/server/public_simplechat/ui.mjs new file mode 100644 index 000000000..b2d5b9aea --- /dev/null +++ b/examples/server/public_simplechat/ui.mjs @@ -0,0 +1,211 @@ +//@ts-check +// Helpers to work with html elements +// by Humans for All +// + + +/** + * Set the class of the children, based on whether it is the idSelected or not. + * @param {HTMLDivElement} elBase + * @param {string} idSelected + * @param {string} classSelected + * @param {string} classUnSelected + */ +export function el_children_config_class(elBase, idSelected, classSelected, classUnSelected="") { + for(let child of elBase.children) { + if (child.id == idSelected) { + child.className = classSelected; + } else { + child.className = classUnSelected; + } + } +} + +/** + * Create button and set it up. + * @param {string} id + * @param {(this: HTMLButtonElement, ev: MouseEvent) => any} callback + * @param {string | undefined} name + * @param {string | undefined} innerText + */ +export function el_create_button(id, callback, name=undefined, innerText=undefined) { + if (!name) { + name = id; + } + if (!innerText) { + innerText = id; + } + let btn = document.createElement("button"); + btn.id = id; + btn.name = name; + btn.innerText = innerText; + btn.addEventListener("click", callback); + return btn; +} + +/** + * Create a para and set it up. Optionaly append it to a passed parent. + * @param {string} text + * @param {HTMLElement | undefined} elParent + * @param {string | undefined} id + */ +export function el_create_append_p(text, elParent=undefined, id=undefined) { + let para = document.createElement("p"); + para.innerText = text; + if (id) { + para.id = id; + } + if (elParent) { + elParent.appendChild(para); + } + return para; +} + +/** + * Create a button which represents bool value using specified text wrt true and false. + * When ever user clicks the button, it will toggle the value and update the shown text. + * + * @param {string} id + * @param {{true: string, false: string}} texts + * @param {boolean} defaultValue + * @param {function(boolean):void} cb + */ +export function el_create_boolbutton(id, texts, defaultValue, cb) { + let el = document.createElement("button"); + el["xbool"] = defaultValue; + el["xtexts"] = structuredClone(texts); + el.innerText = el["xtexts"][String(defaultValue)]; + if (id) { + el.id = id; + } + el.addEventListener('click', (ev)=>{ + el["xbool"] = !el["xbool"]; + el.innerText = el["xtexts"][String(el["xbool"])]; + cb(el["xbool"]); + }) + return el; +} + +/** + * Create a div wrapped button which represents bool value using specified text wrt true and false. + * @param {string} id + * @param {string} label + * @param {{ true: string; false: string; }} texts + * @param {boolean} defaultValue + * @param {(arg0: boolean) => void} cb + * @param {string} className + */ +export function el_creatediv_boolbutton(id, label, texts, defaultValue, cb, className="gridx2") { + let div = document.createElement("div"); + div.className = className; + let lbl = document.createElement("label"); + lbl.setAttribute("for", id); + lbl.innerText = label; + div.appendChild(lbl); + let btn = el_create_boolbutton(id, texts, defaultValue, cb); + div.appendChild(btn); + return { div: div, el: btn }; +} + + +/** + * Create a select ui element, with a set of options to select from. + * * options: an object which contains name-value pairs + * * defaultOption: the value whose name should be choosen, by default. + * * cb : the call back returns the name string of the option selected. + * + * @param {string} id + * @param {Object