2023-03-25 19:26:40 +01:00
|
|
|
#include "common.h"
|
2023-03-24 16:19:05 +01:00
|
|
|
|
2023-03-11 00:04:06 +01:00
|
|
|
#include <cassert>
|
2023-04-30 20:41:35 +02:00
|
|
|
#include <iostream>
|
2023-03-11 00:04:06 +01:00
|
|
|
#include <cstring>
|
2023-03-10 19:40:58 +01:00
|
|
|
#include <fstream>
|
2023-03-12 21:28:36 +01:00
|
|
|
#include <string>
|
2023-03-22 06:32:36 +01:00
|
|
|
#include <iterator>
|
|
|
|
#include <algorithm>
|
llama : new sampling algorithms (#1126)
* Sample interface, new samplers.
New samplers:
- locally typical sampling
- tail free sampling
- frequency and presence penalty
- mirostat
Ignore EOS fix: -inf should be used.
* mirostat
* Added --logit-bias and --no-penalize-nl, removed std::span
* Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
* Save and load example adjust
* Tests
* Windows build fix
* Windows test fix
2023-04-29 07:34:41 +02:00
|
|
|
#include <sstream>
|
2023-04-30 20:41:35 +02:00
|
|
|
|
|
|
|
#if defined(__APPLE__) && defined(__MACH__)
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <sys/sysctl.h>
|
|
|
|
#endif
|
2023-03-10 19:40:58 +01:00
|
|
|
|
2023-03-28 16:09:55 +02:00
|
|
|
#if defined (_WIN32)
|
2023-04-08 17:49:39 +02:00
|
|
|
#include <fcntl.h>
|
|
|
|
#include <io.h>
|
2023-03-28 16:09:55 +02:00
|
|
|
#pragma comment(lib,"kernel32.lib")
|
|
|
|
extern "C" __declspec(dllimport) void* __stdcall GetStdHandle(unsigned long nStdHandle);
|
|
|
|
extern "C" __declspec(dllimport) int __stdcall GetConsoleMode(void* hConsoleHandle, unsigned long* lpMode);
|
|
|
|
extern "C" __declspec(dllimport) int __stdcall SetConsoleMode(void* hConsoleHandle, unsigned long dwMode);
|
|
|
|
extern "C" __declspec(dllimport) int __stdcall SetConsoleCP(unsigned int wCodePageID);
|
|
|
|
extern "C" __declspec(dllimport) int __stdcall SetConsoleOutputCP(unsigned int wCodePageID);
|
2023-04-11 21:45:44 +02:00
|
|
|
extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int CodePage, unsigned long dwFlags,
|
|
|
|
const wchar_t * lpWideCharStr, int cchWideChar,
|
|
|
|
char * lpMultiByteStr, int cbMultiByte,
|
2023-04-08 17:49:39 +02:00
|
|
|
const char * lpDefaultChar, bool * lpUsedDefaultChar);
|
|
|
|
#define CP_UTF8 65001
|
2023-03-28 16:09:55 +02:00
|
|
|
#endif
|
2023-03-12 21:15:00 +01:00
|
|
|
|
2023-04-30 20:41:35 +02:00
|
|
|
int32_t get_num_physical_cores() {
|
2023-03-17 18:47:35 +01:00
|
|
|
#ifdef __linux__
|
|
|
|
std::ifstream cpuinfo("/proc/cpuinfo");
|
2023-04-30 20:41:35 +02:00
|
|
|
std::string line;
|
|
|
|
while (std::getline(cpuinfo, line)) {
|
|
|
|
std::size_t pos = line.find("cpu cores");
|
|
|
|
if (pos != std::string::npos) {
|
|
|
|
pos = line.find(": ", pos);
|
|
|
|
if (pos != std::string::npos) {
|
|
|
|
try {
|
|
|
|
// Extract the number and return it
|
|
|
|
return static_cast<int32_t>(std::stoul(line.substr(pos + 2)));
|
|
|
|
} catch (const std::invalid_argument &) {
|
|
|
|
// Ignore if we could not parse
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#elif defined(__APPLE__) && defined(__MACH__)
|
|
|
|
int32_t num_physical_cores;
|
|
|
|
size_t len = sizeof(num_physical_cores);
|
|
|
|
int result = sysctlbyname("hw.perflevel0.physicalcpu", &num_physical_cores, &len, NULL, 0);
|
|
|
|
if (result == 0) {
|
|
|
|
return num_physical_cores;
|
|
|
|
}
|
|
|
|
result = sysctlbyname("hw.physicalcpu", &num_physical_cores, &len, NULL, 0);
|
|
|
|
if (result == 0) {
|
|
|
|
return num_physical_cores;
|
2023-03-17 18:47:35 +01:00
|
|
|
}
|
2023-04-30 20:41:35 +02:00
|
|
|
#elif defined(_WIN32)
|
|
|
|
//TODO: Implement
|
|
|
|
#endif
|
|
|
|
unsigned int n_threads = std::thread::hardware_concurrency();
|
|
|
|
return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;
|
|
|
|
}
|
2023-03-17 18:47:35 +01:00
|
|
|
|
2023-05-04 14:08:25 +02:00
|
|
|
void process_escapes(std::string& input) {
|
|
|
|
std::size_t input_len = input.length();
|
|
|
|
std::size_t output_idx = 0;
|
2023-05-03 03:46:20 +02:00
|
|
|
|
2023-05-04 14:08:25 +02:00
|
|
|
for (std::size_t input_idx = 0; input_idx < input_len; ++input_idx) {
|
|
|
|
if (input[input_idx] == '\\' && input_idx + 1 < input_len) {
|
|
|
|
switch (input[++input_idx]) {
|
|
|
|
case 'n': input[output_idx++] = '\n'; break;
|
|
|
|
case 'r': input[output_idx++] = '\r'; break;
|
|
|
|
case 't': input[output_idx++] = '\t'; break;
|
|
|
|
case '\'': input[output_idx++] = '\''; break;
|
|
|
|
case '\"': input[output_idx++] = '\"'; break;
|
|
|
|
case '\\': input[output_idx++] = '\\'; break;
|
|
|
|
default: input[output_idx++] = '\\';
|
|
|
|
input[output_idx++] = input[input_idx]; break;
|
2023-05-03 03:46:20 +02:00
|
|
|
}
|
2023-05-04 14:08:25 +02:00
|
|
|
} else {
|
|
|
|
input[output_idx++] = input[input_idx];
|
2023-05-03 03:46:20 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-04 14:08:25 +02:00
|
|
|
input.resize(output_idx);
|
2023-05-03 03:46:20 +02:00
|
|
|
}
|
|
|
|
|
2023-04-30 20:41:35 +02:00
|
|
|
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
|
2023-03-23 18:54:28 +01:00
|
|
|
bool invalid_param = false;
|
2023-05-04 14:08:25 +02:00
|
|
|
bool escape_prompt = false;
|
2023-03-23 18:54:28 +01:00
|
|
|
std::string arg;
|
2023-04-02 04:41:12 +02:00
|
|
|
gpt_params default_params;
|
|
|
|
|
2023-03-10 19:40:58 +01:00
|
|
|
for (int i = 1; i < argc; i++) {
|
2023-03-23 18:54:28 +01:00
|
|
|
arg = argv[i];
|
2023-03-10 19:40:58 +01:00
|
|
|
|
|
|
|
if (arg == "-s" || arg == "--seed") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.seed = std::stoi(argv[i]);
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "-t" || arg == "--threads") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.n_threads = std::stoi(argv[i]);
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "-p" || arg == "--prompt") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-05-04 14:08:25 +02:00
|
|
|
params.prompt = argv[i];
|
|
|
|
} else if (arg == "-e") {
|
|
|
|
escape_prompt = true;
|
2023-04-28 17:59:37 +02:00
|
|
|
} else if (arg == "--session") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.path_session = argv[i];
|
2023-03-12 21:28:36 +01:00
|
|
|
} else if (arg == "-f" || arg == "--file") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
std::ifstream file(argv[i]);
|
2023-03-31 20:03:48 +02:00
|
|
|
if (!file) {
|
2023-04-14 21:58:43 +02:00
|
|
|
fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
|
2023-03-31 20:03:48 +02:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-03-19 17:37:02 +01:00
|
|
|
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
|
2023-03-19 18:04:44 +01:00
|
|
|
if (params.prompt.back() == '\n') {
|
|
|
|
params.prompt.pop_back();
|
|
|
|
}
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "-n" || arg == "--n_predict") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.n_predict = std::stoi(argv[i]);
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "--top_k") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.top_k = std::stoi(argv[i]);
|
2023-03-15 20:42:40 +01:00
|
|
|
} else if (arg == "-c" || arg == "--ctx_size") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.n_ctx = std::stoi(argv[i]);
|
2023-03-24 22:17:37 +01:00
|
|
|
} else if (arg == "--memory_f32") {
|
|
|
|
params.memory_f16 = false;
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "--top_p") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.top_p = std::stof(argv[i]);
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "--temp") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.temp = std::stof(argv[i]);
|
llama : new sampling algorithms (#1126)
* Sample interface, new samplers.
New samplers:
- locally typical sampling
- tail free sampling
- frequency and presence penalty
- mirostat
Ignore EOS fix: -inf should be used.
* mirostat
* Added --logit-bias and --no-penalize-nl, removed std::span
* Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
* Save and load example adjust
* Tests
* Windows build fix
* Windows test fix
2023-04-29 07:34:41 +02:00
|
|
|
} else if (arg == "--tfs") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.tfs_z = std::stof(argv[i]);
|
|
|
|
} else if (arg == "--typical") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.typical_p = std::stof(argv[i]);
|
2023-03-12 10:27:42 +01:00
|
|
|
} else if (arg == "--repeat_last_n") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.repeat_last_n = std::stoi(argv[i]);
|
2023-03-12 10:27:42 +01:00
|
|
|
} else if (arg == "--repeat_penalty") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.repeat_penalty = std::stof(argv[i]);
|
llama : new sampling algorithms (#1126)
* Sample interface, new samplers.
New samplers:
- locally typical sampling
- tail free sampling
- frequency and presence penalty
- mirostat
Ignore EOS fix: -inf should be used.
* mirostat
* Added --logit-bias and --no-penalize-nl, removed std::span
* Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
* Save and load example adjust
* Tests
* Windows build fix
* Windows test fix
2023-04-29 07:34:41 +02:00
|
|
|
} else if (arg == "--frequency_penalty") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.frequency_penalty = std::stof(argv[i]);
|
|
|
|
} else if (arg == "--presence_penalty") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.presence_penalty = std::stof(argv[i]);
|
|
|
|
} else if (arg == "--mirostat") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.mirostat = std::stoi(argv[i]);
|
|
|
|
} else if (arg == "--mirostat_lr") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.mirostat_eta = std::stof(argv[i]);
|
|
|
|
} else if (arg == "--mirostat_ent") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.mirostat_tau = std::stof(argv[i]);
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "-b" || arg == "--batch_size") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.n_batch = std::stoi(argv[i]);
|
2023-03-24 22:17:37 +01:00
|
|
|
params.n_batch = std::min(512, params.n_batch);
|
2023-03-25 20:36:22 +01:00
|
|
|
} else if (arg == "--keep") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-25 20:36:22 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.n_keep = std::stoi(argv[i]);
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "-m" || arg == "--model") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.model = argv[i];
|
2023-04-17 17:28:55 +02:00
|
|
|
} else if (arg == "--lora") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.lora_adapter = argv[i];
|
|
|
|
params.use_mmap = false;
|
|
|
|
} else if (arg == "--lora-base") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
params.lora_base = argv[i];
|
2023-03-12 22:13:28 +01:00
|
|
|
} else if (arg == "-i" || arg == "--interactive") {
|
|
|
|
params.interactive = true;
|
2023-03-24 16:05:13 +01:00
|
|
|
} else if (arg == "--embedding") {
|
|
|
|
params.embedding = true;
|
2023-03-22 18:16:35 +01:00
|
|
|
} else if (arg == "--interactive-first") {
|
2023-04-24 17:45:32 +02:00
|
|
|
params.interactive_first = true;
|
2023-03-19 17:37:02 +01:00
|
|
|
} else if (arg == "-ins" || arg == "--instruct") {
|
2023-04-14 21:58:43 +02:00
|
|
|
params.instruct = true;
|
2023-03-12 22:13:28 +01:00
|
|
|
} else if (arg == "--color") {
|
|
|
|
params.use_color = true;
|
2023-03-24 16:19:05 +01:00
|
|
|
} else if (arg == "--mlock") {
|
|
|
|
params.use_mlock = true;
|
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 21:24:37 +02:00
|
|
|
} else if (arg == "--no-mmap") {
|
|
|
|
params.use_mmap = false;
|
2023-03-24 22:17:37 +01:00
|
|
|
} else if (arg == "--mtest") {
|
|
|
|
params.mem_test = true;
|
2023-03-25 20:36:22 +01:00
|
|
|
} else if (arg == "--verbose-prompt") {
|
2023-03-25 16:16:50 +01:00
|
|
|
params.verbose_prompt = true;
|
2023-03-12 22:13:28 +01:00
|
|
|
} else if (arg == "-r" || arg == "--reverse-prompt") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-04-14 17:19:17 +02:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.antiprompt.push_back(argv[i]);
|
2023-03-21 17:27:42 +01:00
|
|
|
} else if (arg == "--perplexity") {
|
|
|
|
params.perplexity = true;
|
2023-03-19 19:22:48 +01:00
|
|
|
} else if (arg == "--ignore-eos") {
|
llama : new sampling algorithms (#1126)
* Sample interface, new samplers.
New samplers:
- locally typical sampling
- tail free sampling
- frequency and presence penalty
- mirostat
Ignore EOS fix: -inf should be used.
* mirostat
* Added --logit-bias and --no-penalize-nl, removed std::span
* Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
* Save and load example adjust
* Tests
* Windows build fix
* Windows test fix
2023-04-29 07:34:41 +02:00
|
|
|
params.logit_bias[llama_token_eos()] = -INFINITY;
|
|
|
|
} else if (arg == "--no-penalize-nl") {
|
|
|
|
params.penalize_nl = false;
|
|
|
|
} else if (arg == "-l" || arg == "--logit-bias") {
|
|
|
|
if (++i >= argc) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
std::stringstream ss(argv[i]);
|
|
|
|
llama_token key;
|
|
|
|
char sign;
|
|
|
|
std::string value_str;
|
|
|
|
try {
|
|
|
|
if (ss >> key && ss >> sign && std::getline(ss, value_str) && (sign == '+' || sign == '-')) {
|
|
|
|
params.logit_bias[key] = std::stof(value_str) * ((sign == '-') ? -1.0f : 1.0f);
|
|
|
|
} else {
|
|
|
|
throw std::exception();
|
|
|
|
}
|
|
|
|
} catch (const std::exception &e) {
|
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-03-21 16:42:43 +01:00
|
|
|
} else if (arg == "--n_parts") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-03-23 18:54:28 +01:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.n_parts = std::stoi(argv[i]);
|
2023-03-10 19:40:58 +01:00
|
|
|
} else if (arg == "-h" || arg == "--help") {
|
2023-04-14 21:58:43 +02:00
|
|
|
gpt_print_usage(argc, argv, default_params);
|
2023-03-10 19:40:58 +01:00
|
|
|
exit(0);
|
2023-03-19 19:36:19 +01:00
|
|
|
} else if (arg == "--random-prompt") {
|
|
|
|
params.random_prompt = true;
|
2023-03-25 13:03:19 +01:00
|
|
|
} else if (arg == "--in-prefix") {
|
2023-04-14 21:58:43 +02:00
|
|
|
if (++i >= argc) {
|
2023-04-14 17:19:17 +02:00
|
|
|
invalid_param = true;
|
|
|
|
break;
|
|
|
|
}
|
2023-04-14 21:58:43 +02:00
|
|
|
params.input_prefix = argv[i];
|
2023-03-10 19:40:58 +01:00
|
|
|
} else {
|
|
|
|
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
2023-04-14 21:58:43 +02:00
|
|
|
gpt_print_usage(argc, argv, default_params);
|
2023-03-23 18:54:28 +01:00
|
|
|
exit(1);
|
2023-03-10 19:40:58 +01:00
|
|
|
}
|
|
|
|
}
|
2023-03-23 18:54:28 +01:00
|
|
|
if (invalid_param) {
|
|
|
|
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
|
2023-04-14 21:58:43 +02:00
|
|
|
gpt_print_usage(argc, argv, default_params);
|
2023-03-23 18:54:28 +01:00
|
|
|
exit(1);
|
|
|
|
}
|
2023-05-04 14:08:25 +02:00
|
|
|
if (escape_prompt) {
|
|
|
|
process_escapes(params.prompt);
|
|
|
|
}
|
2023-03-10 19:40:58 +01:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2023-04-14 21:58:43 +02:00
|
|
|
void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
|
|
|
|
fprintf(stderr, "usage: %s [options]\n", argv[0]);
|
2023-03-10 19:40:58 +01:00
|
|
|
fprintf(stderr, "\n");
|
|
|
|
fprintf(stderr, "options:\n");
|
|
|
|
fprintf(stderr, " -h, --help show this help message and exit\n");
|
2023-03-12 22:13:28 +01:00
|
|
|
fprintf(stderr, " -i, --interactive run in interactive mode\n");
|
2023-03-22 18:16:35 +01:00
|
|
|
fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n");
|
2023-04-14 21:58:43 +02:00
|
|
|
fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n");
|
2023-03-12 22:13:28 +01:00
|
|
|
fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n");
|
2023-03-22 18:16:35 +01:00
|
|
|
fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n");
|
2023-03-19 20:33:06 +01:00
|
|
|
fprintf(stderr, " specified more than once for multiple prompts).\n");
|
2023-03-12 22:13:28 +01:00
|
|
|
fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n");
|
2023-05-02 18:23:44 +02:00
|
|
|
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for < 0)\n");
|
2023-03-10 19:40:58 +01:00
|
|
|
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
|
|
|
|
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
|
2023-03-19 19:36:19 +01:00
|
|
|
fprintf(stderr, " prompt to start generation with (default: empty)\n");
|
2023-05-04 14:08:25 +02:00
|
|
|
fprintf(stderr, " -e process prompt escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\)\n");
|
2023-04-28 17:59:37 +02:00
|
|
|
fprintf(stderr, " --session FNAME file to cache model state in (may be large!) (default: none)\n");
|
2023-03-19 19:36:19 +01:00
|
|
|
fprintf(stderr, " --random-prompt start with a randomized prompt.\n");
|
2023-03-25 13:03:19 +01:00
|
|
|
fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n");
|
2023-03-12 21:28:36 +01:00
|
|
|
fprintf(stderr, " -f FNAME, --file FNAME\n");
|
|
|
|
fprintf(stderr, " prompt file to start generation.\n");
|
2023-03-28 16:09:55 +02:00
|
|
|
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict);
|
llama : new sampling algorithms (#1126)
* Sample interface, new samplers.
New samplers:
- locally typical sampling
- tail free sampling
- frequency and presence penalty
- mirostat
Ignore EOS fix: -inf should be used.
* mirostat
* Added --logit-bias and --no-penalize-nl, removed std::span
* Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
* Save and load example adjust
* Tests
* Windows build fix
* Windows test fix
2023-04-29 07:34:41 +02:00
|
|
|
fprintf(stderr, " --top_k N top-k sampling (default: %d, 0 = disabled)\n", params.top_k);
|
|
|
|
fprintf(stderr, " --top_p N top-p sampling (default: %.1f, 1.0 = disabled)\n", (double)params.top_p);
|
|
|
|
fprintf(stderr, " --tfs N tail free sampling, parameter z (default: %.1f, 1.0 = disabled)\n", (double)params.tfs_z);
|
|
|
|
fprintf(stderr, " --typical N locally typical sampling, parameter p (default: %.1f, 1.0 = disabled)\n", (double)params.typical_p);
|
|
|
|
fprintf(stderr, " --repeat_last_n N last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)\n", params.repeat_last_n);
|
|
|
|
fprintf(stderr, " --repeat_penalty N penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)\n", (double)params.repeat_penalty);
|
|
|
|
fprintf(stderr, " --presence_penalty N repeat alpha presence penalty (default: %.1f, 0.0 = disabled)\n", (double)params.presence_penalty);
|
|
|
|
fprintf(stderr, " --frequency_penalty N repeat alpha frequency penalty (default: %.1f, 0.0 = disabled)\n", (double)params.frequency_penalty);
|
|
|
|
fprintf(stderr, " --mirostat N use Mirostat sampling.\n");
|
|
|
|
fprintf(stderr, " Top K, Nucleus, Tail Free and Locally Typical samplers are ignored if used.\n");
|
|
|
|
fprintf(stderr, " (default: %d, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)\n", params.mirostat);
|
|
|
|
fprintf(stderr, " --mirostat_lr N Mirostat learning rate, parameter eta (default: %.1f)\n", (double)params.mirostat_eta);
|
|
|
|
fprintf(stderr, " --mirostat_ent N Mirostat target entropy, parameter tau (default: %.1f)\n", (double)params.mirostat_tau);
|
|
|
|
fprintf(stderr, " -l TOKEN_ID(+/-)BIAS, --logit-bias TOKEN_ID(+/-)BIAS\n");
|
|
|
|
fprintf(stderr, " modifies the likelihood of token appearing in the completion,\n");
|
|
|
|
fprintf(stderr, " i.e. `--logit-bias 15043+1` to increase likelihood of token ' Hello',\n");
|
|
|
|
fprintf(stderr, " or `--logit-bias 15043-1` to decrease likelihood of token ' Hello'\n");
|
2023-03-15 20:42:40 +01:00
|
|
|
fprintf(stderr, " -c N, --ctx_size N size of the prompt context (default: %d)\n", params.n_ctx);
|
llama : new sampling algorithms (#1126)
* Sample interface, new samplers.
New samplers:
- locally typical sampling
- tail free sampling
- frequency and presence penalty
- mirostat
Ignore EOS fix: -inf should be used.
* mirostat
* Added --logit-bias and --no-penalize-nl, removed std::span
* Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
Use C++11, clarify llama API documentation, rename Mirostat parameters to --mirostat_lr and --mirostat_ent, add temperature sampling for Mirostat, simplify Mirostat sampling API parameters (removed N and *k)
* Save and load example adjust
* Tests
* Windows build fix
* Windows test fix
2023-04-29 07:34:41 +02:00
|
|
|
fprintf(stderr, " --ignore-eos ignore end of stream token and continue generating (implies --logit-bias 2-inf)\n");
|
|
|
|
fprintf(stderr, " --no-penalize-nl do not penalize newline token\n");
|
2023-03-24 22:17:37 +01:00
|
|
|
fprintf(stderr, " --memory_f32 use f32 instead of f16 for memory key+value\n");
|
2023-03-28 18:48:20 +02:00
|
|
|
fprintf(stderr, " --temp N temperature (default: %.1f)\n", (double)params.temp);
|
2023-03-21 16:42:43 +01:00
|
|
|
fprintf(stderr, " --n_parts N number of model parts (default: -1 = determine from dimensions)\n");
|
2023-03-10 19:40:58 +01:00
|
|
|
fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch);
|
2023-03-21 17:27:42 +01:00
|
|
|
fprintf(stderr, " --perplexity compute perplexity over the prompt\n");
|
2023-03-28 16:09:55 +02:00
|
|
|
fprintf(stderr, " --keep number of tokens to keep from the initial prompt (default: %d, -1 = all)\n", params.n_keep);
|
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 21:24:37 +02:00
|
|
|
if (llama_mlock_supported()) {
|
2023-03-24 16:19:05 +01:00
|
|
|
fprintf(stderr, " --mlock force system to keep model in RAM rather than swapping or compressing\n");
|
|
|
|
}
|
Rewrite loading code to try to satisfy everyone:
- Support all three formats (ggml, ggmf, ggjt). (However, I didn't
include the hack needed to support GPT4All files without conversion.
Those can still be used after converting them with convert.py from my
other PR.)
- Support both mmap and read (mmap is used by default, but can be
disabled with `--no-mmap`, and is automatically disabled for pre-ggjt
files or on platforms where mmap is not supported).
- Support multi-file models like before, but automatically determine the
number of parts rather than requiring `--n_parts`.
- Improve validation and error checking.
- Stop using the per-file type field (f16) entirely in favor of just
relying on the per-tensor type/size fields. This has no immediate
benefit, but makes it easier to experiment with different formats, and
should make it easier to support the new GPTQ-for-LLaMa models in the
future (I have some work in progress on that front).
- Support VirtualLock on Windows (using the same `--mlock` option as on
Unix).
- Indicate loading progress when using mmap + mlock. (Which led me
to the interesting observation that on my Linux machine, with a
warm file cache, mlock actually takes some time, whereas mmap
without mlock starts almost instantly...)
- To help implement this, move mlock support from ggml to the
loading code.
- madvise/PrefetchVirtualMemory support (based on #740)
- Switch from ifstream to the `fopen` family of functions to avoid
unnecessary copying and, when mmap is enabled, allow reusing the same
file descriptor for both metadata reads and mmap (whereas the existing
implementation opens the file a second time to mmap).
- Quantization now produces a single-file output even with multi-file
inputs (not really a feature as much as 'it was easier this way').
Implementation notes:
I tried to factor the code into more discrete pieces than before.
Regarding code style: I tried to follow the code style, but I'm naughty
and used a few advanced C++ features repeatedly:
- Destructors to make it easier to ensure everything gets cleaned up.
- Exceptions. I don't even usually use exceptions when writing C++, and
I can remove them if desired... but here they make the loading code
much more succinct while still properly handling a variety of errors,
ranging from API calls failing to integer overflow and allocation
failure. The exceptions are converted to error codes at the
API boundary.)
Co-authored-by: Pavol Rusnak <pavol@rusnak.io> (for the bit I copied from #740)
2023-04-08 21:24:37 +02:00
|
|
|
if (llama_mmap_supported()) {
|
|
|
|
fprintf(stderr, " --no-mmap do not memory-map model (slower load but may reduce pageouts if not using mlock)\n");
|
|
|
|
}
|
2023-03-24 22:17:37 +01:00
|
|
|
fprintf(stderr, " --mtest compute maximum memory usage\n");
|
2023-03-25 16:16:50 +01:00
|
|
|
fprintf(stderr, " --verbose-prompt print prompt before generation\n");
|
2023-04-17 17:28:55 +02:00
|
|
|
fprintf(stderr, " --lora FNAME apply LoRA adapter (implies --no-mmap)\n");
|
|
|
|
fprintf(stderr, " --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n");
|
2023-03-10 19:40:58 +01:00
|
|
|
fprintf(stderr, " -m FNAME, --model FNAME\n");
|
|
|
|
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
|
|
|
|
fprintf(stderr, "\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string gpt_random_prompt(std::mt19937 & rng) {
|
|
|
|
const int r = rng() % 10;
|
|
|
|
switch (r) {
|
|
|
|
case 0: return "So";
|
|
|
|
case 1: return "Once upon a time";
|
|
|
|
case 2: return "When";
|
|
|
|
case 3: return "The";
|
|
|
|
case 4: return "After";
|
|
|
|
case 5: return "If";
|
|
|
|
case 6: return "import";
|
|
|
|
case 7: return "He";
|
|
|
|
case 8: return "She";
|
|
|
|
case 9: return "They";
|
|
|
|
default: return "To";
|
|
|
|
}
|
|
|
|
|
|
|
|
return "The";
|
|
|
|
}
|
|
|
|
|
2023-03-22 06:32:36 +01:00
|
|
|
// TODO: not great allocating this every time
|
|
|
|
std::vector<llama_token> llama_tokenize(struct llama_context * ctx, const std::string & text, bool add_bos) {
|
2023-03-22 17:09:38 +01:00
|
|
|
// initialize to prompt numer of chars, since n_tokens <= n_prompt_chars
|
|
|
|
std::vector<llama_token> res(text.size() + (int)add_bos);
|
2023-03-22 06:32:36 +01:00
|
|
|
int n = llama_tokenize(ctx, text.c_str(), res.data(), res.size(), add_bos);
|
2023-03-22 17:09:38 +01:00
|
|
|
assert(n >= 0);
|
2023-03-22 06:32:36 +01:00
|
|
|
res.resize(n);
|
2023-03-10 19:40:58 +01:00
|
|
|
|
2023-03-22 06:32:36 +01:00
|
|
|
return res;
|
2023-03-10 19:40:58 +01:00
|
|
|
}
|
2023-03-28 16:09:55 +02:00
|
|
|
|
2023-05-02 22:39:51 +02:00
|
|
|
struct llama_context * llama_init_from_gpt_params(const gpt_params & params) {
|
|
|
|
auto lparams = llama_context_default_params();
|
|
|
|
|
|
|
|
lparams.n_ctx = params.n_ctx;
|
|
|
|
lparams.n_parts = params.n_parts;
|
|
|
|
lparams.seed = params.seed;
|
|
|
|
lparams.f16_kv = params.memory_f16;
|
|
|
|
lparams.use_mmap = params.use_mmap;
|
|
|
|
lparams.use_mlock = params.use_mlock;
|
2023-05-03 01:36:45 +02:00
|
|
|
lparams.logits_all = params.perplexity;
|
|
|
|
lparams.embedding = params.embedding;
|
2023-05-02 22:39:51 +02:00
|
|
|
|
|
|
|
llama_context * lctx = llama_init_from_file(params.model.c_str(), lparams);
|
|
|
|
|
|
|
|
if (lctx == NULL) {
|
|
|
|
fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str());
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!params.lora_adapter.empty()) {
|
|
|
|
int err = llama_apply_lora_from_file(lctx,
|
|
|
|
params.lora_adapter.c_str(),
|
|
|
|
params.lora_base.empty() ? NULL : params.lora_base.c_str(),
|
|
|
|
params.n_threads);
|
|
|
|
if (err != 0) {
|
|
|
|
fprintf(stderr, "%s: error: failed to apply lora adapter\n", __func__);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return lctx;
|
|
|
|
}
|
|
|
|
|
2023-03-28 16:09:55 +02:00
|
|
|
/* Keep track of current color of output, and emit ANSI code if it changes. */
|
|
|
|
void set_console_color(console_state & con_st, console_color_t color) {
|
|
|
|
if (con_st.use_color && con_st.color != color) {
|
|
|
|
switch(color) {
|
|
|
|
case CONSOLE_COLOR_DEFAULT:
|
|
|
|
printf(ANSI_COLOR_RESET);
|
|
|
|
break;
|
|
|
|
case CONSOLE_COLOR_PROMPT:
|
|
|
|
printf(ANSI_COLOR_YELLOW);
|
|
|
|
break;
|
|
|
|
case CONSOLE_COLOR_USER_INPUT:
|
|
|
|
printf(ANSI_BOLD ANSI_COLOR_GREEN);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
con_st.color = color;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#if defined (_WIN32)
|
|
|
|
void win32_console_init(bool enable_color) {
|
|
|
|
unsigned long dwMode = 0;
|
|
|
|
void* hConOut = GetStdHandle((unsigned long)-11); // STD_OUTPUT_HANDLE (-11)
|
|
|
|
if (!hConOut || hConOut == (void*)-1 || !GetConsoleMode(hConOut, &dwMode)) {
|
|
|
|
hConOut = GetStdHandle((unsigned long)-12); // STD_ERROR_HANDLE (-12)
|
|
|
|
if (hConOut && (hConOut == (void*)-1 || !GetConsoleMode(hConOut, &dwMode))) {
|
|
|
|
hConOut = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (hConOut) {
|
|
|
|
// Enable ANSI colors on Windows 10+
|
|
|
|
if (enable_color && !(dwMode & 0x4)) {
|
|
|
|
SetConsoleMode(hConOut, dwMode | 0x4); // ENABLE_VIRTUAL_TERMINAL_PROCESSING (0x4)
|
|
|
|
}
|
|
|
|
// Set console output codepage to UTF8
|
2023-04-08 17:49:39 +02:00
|
|
|
SetConsoleOutputCP(CP_UTF8);
|
2023-03-28 16:09:55 +02:00
|
|
|
}
|
|
|
|
void* hConIn = GetStdHandle((unsigned long)-10); // STD_INPUT_HANDLE (-10)
|
|
|
|
if (hConIn && hConIn != (void*)-1 && GetConsoleMode(hConIn, &dwMode)) {
|
2023-04-08 17:49:39 +02:00
|
|
|
// Set console input codepage to UTF16
|
|
|
|
_setmode(_fileno(stdin), _O_WTEXT);
|
2023-03-28 16:09:55 +02:00
|
|
|
}
|
|
|
|
}
|
2023-04-08 17:49:39 +02:00
|
|
|
|
|
|
|
// Convert a wide Unicode string to an UTF8 string
|
|
|
|
void win32_utf8_encode(const std::wstring & wstr, std::string & str) {
|
2023-04-11 21:45:44 +02:00
|
|
|
int size_needed = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL);
|
|
|
|
std::string strTo(size_needed, 0);
|
|
|
|
WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &strTo[0], size_needed, NULL, NULL);
|
|
|
|
str = strTo;
|
2023-04-08 17:49:39 +02:00
|
|
|
}
|
2023-03-28 16:09:55 +02:00
|
|
|
#endif
|