2024-01-12 06:59:57 +01:00
|
|
|
#include "common.h"
|
|
|
|
#include "llama.h"
|
|
|
|
|
|
|
|
#include <cmath>
|
|
|
|
#include <cstdio>
|
|
|
|
#include <cstring>
|
|
|
|
#include <ctime>
|
|
|
|
#include <sstream>
|
|
|
|
#include <thread>
|
|
|
|
#include <mutex>
|
|
|
|
#include <vector>
|
|
|
|
#include <fstream>
|
|
|
|
#include <unordered_map>
|
|
|
|
#include <algorithm>
|
|
|
|
|
|
|
|
#if defined(_MSC_VER)
|
|
|
|
#pragma warning(disable: 4244 4267) // possible loss of data
|
|
|
|
#endif
|
|
|
|
|
|
|
|
struct Stats {
|
|
|
|
std::vector<float> values;
|
2024-05-08 02:24:16 +02:00
|
|
|
std::vector<int> counts;
|
2024-01-12 06:59:57 +01:00
|
|
|
int ncall = 0;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct StatParams {
|
2024-04-26 20:06:33 +02:00
|
|
|
std::string dataset;
|
2024-01-12 06:59:57 +01:00
|
|
|
std::string ofile = "imatrix.dat";
|
|
|
|
int n_output_frequency = 10;
|
|
|
|
int verbosity = 1;
|
2024-01-22 13:18:43 +01:00
|
|
|
int keep_every = 0;
|
2024-01-12 06:59:57 +01:00
|
|
|
bool collect_output_weight = false;
|
|
|
|
};
|
|
|
|
|
|
|
|
class IMatrixCollector {
|
|
|
|
public:
|
|
|
|
IMatrixCollector() = default;
|
|
|
|
void set_parameters(StatParams&& params) { m_params = std::move(params); }
|
2024-01-17 17:46:30 +01:00
|
|
|
bool collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data);
|
2024-01-12 06:59:57 +01:00
|
|
|
void save_imatrix() const;
|
2024-02-04 09:39:58 +01:00
|
|
|
bool load_imatrix(const char * file_name, bool add);
|
|
|
|
static bool load_imatrix(const char * file_name, std::unordered_map<std::string, Stats>& imatrix);
|
2024-01-12 06:59:57 +01:00
|
|
|
private:
|
|
|
|
std::unordered_map<std::string, Stats> m_stats;
|
|
|
|
StatParams m_params;
|
|
|
|
std::mutex m_mutex;
|
|
|
|
int m_last_call = 0;
|
2024-01-17 17:46:30 +01:00
|
|
|
std::vector<float> m_src1_data;
|
2024-04-18 15:18:48 +02:00
|
|
|
std::vector<char> m_ids; // the expert ids from ggml_mul_mat_id
|
2024-01-22 13:18:43 +01:00
|
|
|
//
|
2024-04-26 20:06:33 +02:00
|
|
|
void save_imatrix(const char * file_name, const char * dataset) const;
|
2024-01-22 13:18:43 +01:00
|
|
|
void keep_imatrix(int ncall) const;
|
2024-01-12 06:59:57 +01:00
|
|
|
};
|
|
|
|
|
2024-03-24 15:18:45 +01:00
|
|
|
// remove any prefix and suffixes from the name
|
|
|
|
// CUDA0#blk.0.attn_k.weight#0 => blk.0.attn_k.weight
|
|
|
|
static std::string filter_tensor_name(const char * name) {
|
|
|
|
std::string wname;
|
|
|
|
const char * p = strchr(name, '#');
|
|
|
|
if (p != NULL) {
|
|
|
|
p = p + 1;
|
|
|
|
const char * q = strchr(p, '#');
|
|
|
|
if (q != NULL) {
|
|
|
|
wname = std::string(p, q - p);
|
|
|
|
} else {
|
|
|
|
wname = p;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
wname = name;
|
|
|
|
}
|
|
|
|
return wname;
|
|
|
|
}
|
|
|
|
|
2024-01-17 17:46:30 +01:00
|
|
|
bool IMatrixCollector::collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data) {
|
|
|
|
GGML_UNUSED(user_data);
|
|
|
|
|
|
|
|
const struct ggml_tensor * src0 = t->src[0];
|
|
|
|
const struct ggml_tensor * src1 = t->src[1];
|
2024-03-24 15:18:45 +01:00
|
|
|
std::string wname = filter_tensor_name(src0->name);
|
2024-03-18 11:03:04 +01:00
|
|
|
|
2024-01-17 17:46:30 +01:00
|
|
|
// when ask is true, the scheduler wants to know if we are interested in data from this tensor
|
|
|
|
// if we return true, a follow-up call will be made with ask=false in which we can do the actual collection
|
|
|
|
if (ask) {
|
|
|
|
if (t->op == GGML_OP_MUL_MAT_ID) return true; // collect all indirect matrix multiplications
|
|
|
|
if (t->op != GGML_OP_MUL_MAT) return false;
|
2024-04-18 15:18:48 +02:00
|
|
|
// why are small batches ignored (<16 tokens)?
|
2024-01-17 17:46:30 +01:00
|
|
|
if (src1->ne[1] < 16 || src1->type != GGML_TYPE_F32) return false;
|
2024-03-18 11:03:04 +01:00
|
|
|
if (!(wname.substr(0, 4) == "blk." || (m_params.collect_output_weight && wname == "output.weight"))) return false;
|
2024-01-17 17:46:30 +01:00
|
|
|
return true;
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
2024-01-17 17:46:30 +01:00
|
|
|
|
|
|
|
std::lock_guard<std::mutex> lock(m_mutex);
|
|
|
|
|
|
|
|
// copy the data from the GPU memory if needed
|
|
|
|
const bool is_host = ggml_backend_buffer_is_host(src1->buffer);
|
|
|
|
|
|
|
|
if (!is_host) {
|
|
|
|
m_src1_data.resize(ggml_nelements(src1));
|
|
|
|
ggml_backend_tensor_get(src1, m_src1_data.data(), 0, ggml_nbytes(src1));
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
2024-01-17 17:46:30 +01:00
|
|
|
|
|
|
|
const float * data = is_host ? (const float *) src1->data : m_src1_data.data();
|
|
|
|
|
2024-04-03 15:07:05 +02:00
|
|
|
// this has been adapted to the new format of storing merged experts in a single 3d tensor
|
|
|
|
// ref: https://github.com/ggerganov/llama.cpp/pull/6387
|
2024-01-17 17:46:30 +01:00
|
|
|
if (t->op == GGML_OP_MUL_MAT_ID) {
|
2024-04-18 15:18:48 +02:00
|
|
|
// ids -> [n_experts_used, n_tokens]
|
|
|
|
// src1 -> [cols, n_expert_used, n_tokens]
|
2024-04-03 15:07:05 +02:00
|
|
|
const ggml_tensor * ids = t->src[2];
|
|
|
|
const int n_as = src0->ne[2];
|
2024-04-18 15:18:48 +02:00
|
|
|
const int n_ids = ids->ne[0];
|
2024-01-17 17:46:30 +01:00
|
|
|
|
2024-04-03 15:07:05 +02:00
|
|
|
// the top-k selected expert ids are stored in the ids tensor
|
|
|
|
// for simplicity, always copy ids to host, because it is small
|
2024-04-18 15:18:48 +02:00
|
|
|
// take into account that ids is not contiguous!
|
|
|
|
|
|
|
|
GGML_ASSERT(ids->ne[1] == src1->ne[2]);
|
|
|
|
|
|
|
|
m_ids.resize(ggml_nbytes(ids));
|
2024-04-03 15:07:05 +02:00
|
|
|
ggml_backend_tensor_get(ids, m_ids.data(), 0, ggml_nbytes(ids));
|
|
|
|
|
|
|
|
auto & e = m_stats[wname];
|
|
|
|
|
|
|
|
++e.ncall;
|
2024-01-17 17:46:30 +01:00
|
|
|
|
2024-04-18 15:18:48 +02:00
|
|
|
if (e.values.empty()) {
|
|
|
|
e.values.resize(src1->ne[0]*n_as, 0);
|
2024-05-08 02:24:16 +02:00
|
|
|
e.counts.resize(src1->ne[0]*n_as, 0);
|
2024-04-18 15:18:48 +02:00
|
|
|
}
|
|
|
|
else if (e.values.size() != (size_t)src1->ne[0]*n_as) {
|
|
|
|
fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", wname.c_str(), (int)e.values.size(), (int)src1->ne[0]*n_as);
|
|
|
|
exit(1); //GGML_ASSERT(false);
|
|
|
|
}
|
|
|
|
if (m_params.verbosity > 1) {
|
|
|
|
printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, wname.c_str(), ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[2], (int)src1->type);
|
|
|
|
}
|
2024-01-17 17:46:30 +01:00
|
|
|
// loop over all possible experts, regardless if they are used or not in the batch
|
|
|
|
for (int ex = 0; ex < n_as; ++ex) {
|
2024-04-03 15:07:05 +02:00
|
|
|
size_t e_start = ex*src1->ne[0];
|
2024-04-18 15:18:48 +02:00
|
|
|
|
|
|
|
for (int idx = 0; idx < n_ids; ++idx) {
|
|
|
|
for (int row = 0; row < (int)src1->ne[2]; ++row) {
|
|
|
|
const int excur = *(const int32_t *) (m_ids.data() + row*ids->nb[1] + idx*ids->nb[0]);
|
|
|
|
|
|
|
|
GGML_ASSERT(excur >= 0 && excur < n_as); // sanity check
|
|
|
|
|
|
|
|
if (excur != ex) continue;
|
|
|
|
|
|
|
|
const int64_t i11 = idx % src1->ne[1];
|
|
|
|
const int64_t i12 = row;
|
|
|
|
const float * x = (const float *)((const char *)data + i11*src1->nb[1] + i12*src1->nb[2]);
|
|
|
|
|
|
|
|
for (int j = 0; j < (int)src1->ne[0]; ++j) {
|
|
|
|
e.values[e_start + j] += x[j]*x[j];
|
2024-05-08 02:24:16 +02:00
|
|
|
e.counts[e_start + j]++;
|
2024-04-18 15:18:48 +02:00
|
|
|
}
|
2024-01-17 17:46:30 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (e.ncall > m_last_call) {
|
|
|
|
m_last_call = e.ncall;
|
|
|
|
if (m_last_call % m_params.n_output_frequency == 0) {
|
|
|
|
save_imatrix();
|
|
|
|
}
|
2024-01-22 13:18:43 +01:00
|
|
|
if (m_params.keep_every > 0 && m_last_call%m_params.keep_every == 0) {
|
|
|
|
keep_imatrix(m_last_call);
|
|
|
|
}
|
2024-01-17 17:46:30 +01:00
|
|
|
}
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
2024-01-17 17:46:30 +01:00
|
|
|
} else {
|
2024-03-18 11:03:04 +01:00
|
|
|
auto& e = m_stats[wname];
|
2024-01-17 17:46:30 +01:00
|
|
|
if (e.values.empty()) {
|
|
|
|
e.values.resize(src1->ne[0], 0);
|
2024-05-08 02:24:16 +02:00
|
|
|
e.counts.resize(src1->ne[0], 0);
|
2024-01-17 17:46:30 +01:00
|
|
|
}
|
|
|
|
else if (e.values.size() != (size_t)src1->ne[0]) {
|
2024-03-18 11:03:04 +01:00
|
|
|
fprintf(stderr, "Oops: inconsistent size for %s (%d vs %d)\n", wname.c_str(), (int)e.values.size(), (int)src1->ne[0]);
|
2024-01-17 17:46:30 +01:00
|
|
|
exit(1); //GGML_ASSERT(false);
|
|
|
|
}
|
|
|
|
++e.ncall;
|
|
|
|
if (m_params.verbosity > 1) {
|
2024-03-18 11:03:04 +01:00
|
|
|
printf("%s[%d]: %32s, %s, %5d x %5d, %d\n", __func__, m_last_call, wname.c_str(), ggml_op_name(t->op), (int)src1->ne[0], (int)src1->ne[1], (int)src1->type);
|
2024-01-17 17:46:30 +01:00
|
|
|
}
|
|
|
|
for (int row = 0; row < (int)src1->ne[1]; ++row) {
|
|
|
|
const float * x = data + row * src1->ne[0];
|
|
|
|
for (int j = 0; j < (int)src1->ne[0]; ++j) {
|
|
|
|
e.values[j] += x[j]*x[j];
|
2024-05-08 02:24:16 +02:00
|
|
|
e.counts[j]++;
|
2024-01-17 17:46:30 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (e.ncall > m_last_call) {
|
|
|
|
m_last_call = e.ncall;
|
|
|
|
if (m_last_call % m_params.n_output_frequency == 0) {
|
|
|
|
save_imatrix();
|
|
|
|
}
|
2024-01-22 13:18:43 +01:00
|
|
|
if (m_params.keep_every > 0 && m_last_call%m_params.keep_every == 0) {
|
|
|
|
keep_imatrix(m_last_call);
|
|
|
|
}
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
}
|
2024-01-17 17:46:30 +01:00
|
|
|
|
|
|
|
return true;
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void IMatrixCollector::save_imatrix() const {
|
2024-04-26 20:06:33 +02:00
|
|
|
save_imatrix(m_params.ofile.empty() ? "imatrix.dat" : m_params.ofile.c_str(), m_params.dataset.c_str());
|
2024-01-22 13:18:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
void IMatrixCollector::keep_imatrix(int ncall) const {
|
|
|
|
auto file_name = m_params.ofile;
|
|
|
|
if (file_name.empty()) file_name = "imatrix.dat";
|
|
|
|
file_name += ".at_";
|
|
|
|
file_name += std::to_string(ncall);
|
2024-04-26 20:06:33 +02:00
|
|
|
save_imatrix(file_name.c_str(), m_params.dataset.c_str());
|
2024-01-22 13:18:43 +01:00
|
|
|
}
|
|
|
|
|
2024-04-26 20:06:33 +02:00
|
|
|
void IMatrixCollector::save_imatrix(const char * fname, const char * dataset) const {
|
2024-01-12 06:59:57 +01:00
|
|
|
std::ofstream out(fname, std::ios::binary);
|
|
|
|
int n_entries = m_stats.size();
|
2024-04-26 20:06:33 +02:00
|
|
|
out.write((const char *) &n_entries, sizeof(n_entries));
|
|
|
|
for (const auto & p : m_stats) {
|
2024-01-12 06:59:57 +01:00
|
|
|
int len = p.first.size();
|
2024-04-26 20:06:33 +02:00
|
|
|
out.write((const char *) &len, sizeof(len));
|
2024-01-12 06:59:57 +01:00
|
|
|
out.write(p.first.c_str(), len);
|
2024-04-26 20:06:33 +02:00
|
|
|
out.write((const char *) &p.second.ncall, sizeof(p.second.ncall));
|
2024-01-12 06:59:57 +01:00
|
|
|
int nval = p.second.values.size();
|
2024-04-26 20:06:33 +02:00
|
|
|
out.write((const char *) &nval, sizeof(nval));
|
2024-05-08 02:24:16 +02:00
|
|
|
if (nval > 0) {
|
|
|
|
std::vector<float> tmp(nval);
|
|
|
|
for (int i = 0; i < nval; i++) {
|
|
|
|
tmp[i] = (p.second.values[i] / static_cast<float>(p.second.counts[i])) * static_cast<float>(p.second.ncall);
|
|
|
|
}
|
|
|
|
out.write((const char*)tmp.data(), nval*sizeof(float));
|
|
|
|
}
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
2024-04-26 20:06:33 +02:00
|
|
|
|
|
|
|
// Write the number of call the matrix was computed with
|
|
|
|
out.write((const char *) &m_last_call, sizeof(m_last_call));
|
|
|
|
|
|
|
|
// Write the dataset name at the end of the file to later on specify it in quantize
|
|
|
|
int n_dataset = strlen(dataset);
|
|
|
|
out.write((const char *) &n_dataset, sizeof(n_dataset));
|
|
|
|
out.write(dataset, n_dataset);
|
|
|
|
|
2024-01-12 06:59:57 +01:00
|
|
|
if (m_params.verbosity > 0) {
|
2024-04-26 20:06:33 +02:00
|
|
|
fprintf(stderr, "\n%s: stored collected data after %d chunks in %s\n", __func__, m_last_call, fname);
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-04 09:39:58 +01:00
|
|
|
bool IMatrixCollector::load_imatrix(const char * imatrix_file, std::unordered_map<std::string, Stats>& imatrix_data) {
|
|
|
|
std::ifstream in(imatrix_file, std::ios::binary);
|
|
|
|
if (!in) {
|
|
|
|
printf("%s: failed to open %s\n",__func__,imatrix_file);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
int n_entries;
|
|
|
|
in.read((char*)&n_entries, sizeof(n_entries));
|
|
|
|
if (in.fail() || n_entries < 1) {
|
|
|
|
printf("%s: no data in file %s\n", __func__, imatrix_file);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
for (int i = 0; i < n_entries; ++i) {
|
|
|
|
int len; in.read((char *)&len, sizeof(len));
|
|
|
|
std::vector<char> name_as_vec(len+1);
|
|
|
|
in.read((char *)name_as_vec.data(), len);
|
|
|
|
if (in.fail()) {
|
|
|
|
printf("%s: failed reading name for entry %d from %s\n",__func__,i+1,imatrix_file);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
name_as_vec[len] = 0;
|
|
|
|
std::string name{name_as_vec.data()};
|
|
|
|
auto& e = imatrix_data[std::move(name)];
|
|
|
|
int ncall;
|
|
|
|
in.read((char*)&ncall, sizeof(ncall));
|
|
|
|
int nval;
|
|
|
|
in.read((char *)&nval, sizeof(nval));
|
|
|
|
if (in.fail() || nval < 1) {
|
|
|
|
printf("%s: failed reading number of values for entry %d\n",__func__,i);
|
|
|
|
imatrix_data = {};
|
|
|
|
return false;
|
|
|
|
}
|
2024-05-08 02:24:16 +02:00
|
|
|
|
|
|
|
// When re-called from load_imatrix() with add set, this will already be created.
|
|
|
|
if (e.values.empty()) {
|
|
|
|
e.values.resize(nval, 0);
|
|
|
|
e.counts.resize(nval, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<float> tmp(nval);
|
|
|
|
in.read((char*)tmp.data(), nval*sizeof(float));
|
2024-02-04 09:39:58 +01:00
|
|
|
if (in.fail()) {
|
|
|
|
printf("%s: failed reading data for entry %d\n",__func__,i);
|
|
|
|
imatrix_data = {};
|
|
|
|
return false;
|
|
|
|
}
|
2024-05-08 02:24:16 +02:00
|
|
|
|
|
|
|
// Recreate the state as expected by save_imatrix(), and corerct for weighted sum.
|
|
|
|
for (int i = 0; i < nval; i++) {
|
|
|
|
e.values[i] += tmp[i];
|
|
|
|
e.counts[i] += ncall;
|
|
|
|
}
|
|
|
|
e.ncall += ncall;
|
|
|
|
|
2024-02-04 09:39:58 +01:00
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool IMatrixCollector::load_imatrix(const char * file_name, bool add) {
|
|
|
|
if (!add) {
|
|
|
|
m_stats.clear();
|
|
|
|
}
|
|
|
|
return load_imatrix(file_name, m_stats);
|
|
|
|
}
|
|
|
|
|
2024-01-12 06:59:57 +01:00
|
|
|
static IMatrixCollector g_collector;
|
|
|
|
|
2024-01-17 17:46:30 +01:00
|
|
|
static bool ik_collect_imatrix(struct ggml_tensor * t, bool ask, void * user_data) {
|
|
|
|
return g_collector.collect_imatrix(t, ask, user_data);
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
struct results_log_softmax {
|
|
|
|
double log_softmax;
|
|
|
|
float logit;
|
|
|
|
float prob;
|
|
|
|
};
|
|
|
|
|
|
|
|
static std::vector<float> softmax(const std::vector<float>& logits) {
|
|
|
|
std::vector<float> probs(logits.size());
|
|
|
|
float max_logit = logits[0];
|
|
|
|
for (float v : logits) {
|
|
|
|
max_logit = std::max(max_logit, v);
|
|
|
|
}
|
|
|
|
double sum_exp = 0.0;
|
|
|
|
for (size_t i = 0; i < logits.size(); i++) {
|
|
|
|
// Subtract the maximum logit value from the current logit value for numerical stability
|
|
|
|
const float logit = logits[i] - max_logit;
|
|
|
|
const float exp_logit = expf(logit);
|
|
|
|
sum_exp += exp_logit;
|
|
|
|
probs[i] = exp_logit;
|
|
|
|
}
|
|
|
|
for (size_t i = 0; i < probs.size(); i++) {
|
|
|
|
probs[i] /= sum_exp;
|
|
|
|
}
|
|
|
|
return probs;
|
|
|
|
}
|
|
|
|
|
|
|
|
static results_log_softmax log_softmax(int n_vocab, const float * logits, int tok) {
|
|
|
|
float max_logit = logits[0];
|
|
|
|
for (int i = 1; i < n_vocab; ++i) {
|
|
|
|
max_logit = std::max(max_logit, logits[i]);
|
|
|
|
}
|
|
|
|
double sum_exp = 0.0;
|
|
|
|
for (int i = 0; i < n_vocab; ++i) {
|
|
|
|
sum_exp += expf(logits[i] - max_logit);
|
|
|
|
}
|
|
|
|
return {logits[tok] - max_logit - log(sum_exp), logits[tok], expf(logits[tok] - max_logit) / (float) sum_exp};
|
|
|
|
}
|
|
|
|
|
|
|
|
static void process_logits(
|
|
|
|
int n_vocab, const float * logits, const int * tokens, int n_token, std::vector<std::thread> & workers,
|
|
|
|
double & nll, double & nll2, float * logit_history, float * prob_history
|
|
|
|
) {
|
|
|
|
std::mutex mutex;
|
|
|
|
int counter = 0;
|
|
|
|
auto compute = [&mutex, &counter, &nll, &nll2, logit_history, prob_history, n_vocab, logits, tokens, n_token] () {
|
|
|
|
double local_nll = 0;
|
|
|
|
double local_nll2 = 0;
|
|
|
|
while (true) {
|
|
|
|
std::unique_lock<std::mutex> lock(mutex);
|
|
|
|
int i = counter++;
|
|
|
|
if (i >= n_token) {
|
|
|
|
nll += local_nll; nll2 += local_nll2;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
lock.unlock();
|
|
|
|
const results_log_softmax results = log_softmax(n_vocab, logits + i*n_vocab, tokens[i+1]);
|
|
|
|
const double v = -results.log_softmax;
|
|
|
|
local_nll += v;
|
|
|
|
local_nll2 += v*v;
|
|
|
|
|
|
|
|
logit_history[i] = results.logit;
|
|
|
|
prob_history[i] = results.prob;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
for (auto & w : workers) {
|
|
|
|
w = std::thread(compute);
|
|
|
|
}
|
|
|
|
compute();
|
|
|
|
for (auto & w : workers) {
|
|
|
|
w.join();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-04 09:39:58 +01:00
|
|
|
static bool compute_imatrix(llama_context * ctx, const gpt_params & params, bool compute_ppl, int from_chunk) {
|
2024-01-12 06:59:57 +01:00
|
|
|
|
|
|
|
const bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));
|
2024-04-09 19:44:08 +02:00
|
|
|
GGML_ASSERT(llama_add_eos_token(llama_get_model(ctx)) != 1);
|
2024-01-12 06:59:57 +01:00
|
|
|
const int n_ctx = llama_n_ctx(ctx);
|
|
|
|
|
|
|
|
auto tim1 = std::chrono::high_resolution_clock::now();
|
|
|
|
fprintf(stderr, "%s: tokenizing the input ..\n", __func__);
|
|
|
|
|
2024-04-09 19:44:08 +02:00
|
|
|
std::vector<llama_token> tokens = ::llama_tokenize(ctx, params.prompt, true);
|
2024-01-12 06:59:57 +01:00
|
|
|
|
|
|
|
auto tim2 = std::chrono::high_resolution_clock::now();
|
|
|
|
fprintf(stderr, "%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast<std::chrono::microseconds>(tim2-tim1).count());
|
|
|
|
|
2024-02-04 09:39:58 +01:00
|
|
|
if (from_chunk > 0) {
|
|
|
|
if (size_t((from_chunk + 2)*n_ctx) >= tokens.size()) {
|
|
|
|
fprintf(stderr, "%s: there will be not enough tokens left after removing %d chunks\n", __func__, from_chunk);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
fprintf(stderr, "%s: removing initial %d chunks (%d tokens)\n", __func__, from_chunk, from_chunk*n_ctx);
|
|
|
|
tokens.erase(tokens.begin(), tokens.begin() + from_chunk*n_ctx);
|
|
|
|
}
|
|
|
|
|
2024-01-12 06:59:57 +01:00
|
|
|
if (int(tokens.size()) < 2*n_ctx) {
|
|
|
|
fprintf(stderr, "%s: you need at least %d tokens for a context of %d tokens\n",__func__,2*n_ctx,
|
|
|
|
n_ctx);
|
|
|
|
fprintf(stderr, "%s: the data file you provided tokenizes to only %zu tokens\n",__func__,tokens.size());
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<float> logit_history;
|
|
|
|
std::vector<float> prob_history;
|
2024-01-21 07:01:20 +01:00
|
|
|
|
|
|
|
if (compute_ppl) {
|
|
|
|
logit_history.resize(tokens.size());
|
|
|
|
prob_history.resize(tokens.size());
|
|
|
|
}
|
2024-01-12 06:59:57 +01:00
|
|
|
|
|
|
|
const int n_chunk_max = tokens.size() / n_ctx;
|
|
|
|
|
|
|
|
const int n_chunk = params.n_chunks < 0 ? n_chunk_max : std::min(params.n_chunks, n_chunk_max);
|
|
|
|
const int n_vocab = llama_n_vocab(llama_get_model(ctx));
|
|
|
|
const int n_batch = params.n_batch;
|
|
|
|
|
|
|
|
int count = 0;
|
|
|
|
double nll = 0.0;
|
|
|
|
double nll2 = 0.0;
|
|
|
|
|
|
|
|
fprintf(stderr, "%s: computing over %d chunks with batch_size %d\n", __func__, n_chunk, n_batch);
|
|
|
|
|
|
|
|
std::vector<std::thread> workers(std::thread::hardware_concurrency() - 1);
|
|
|
|
|
2024-01-21 07:01:20 +01:00
|
|
|
const int num_batches = (n_ctx + n_batch - 1) / n_batch;
|
|
|
|
|
|
|
|
std::vector<float> logits;
|
|
|
|
if (compute_ppl && num_batches > 1) {
|
|
|
|
logits.reserve((size_t)n_ctx * n_vocab);
|
|
|
|
}
|
|
|
|
|
2024-01-12 06:59:57 +01:00
|
|
|
for (int i = 0; i < n_chunk; ++i) {
|
|
|
|
const int start = i * n_ctx;
|
|
|
|
const int end = start + n_ctx;
|
|
|
|
|
|
|
|
std::vector<float> logits;
|
|
|
|
|
|
|
|
const auto t_start = std::chrono::high_resolution_clock::now();
|
|
|
|
|
|
|
|
// clear the KV cache
|
|
|
|
llama_kv_cache_clear(ctx);
|
|
|
|
|
|
|
|
for (int j = 0; j < num_batches; ++j) {
|
|
|
|
const int batch_start = start + j * n_batch;
|
|
|
|
const int batch_size = std::min(end - batch_start, n_batch);
|
|
|
|
|
|
|
|
// save original token and restore it after eval
|
|
|
|
const auto token_org = tokens[batch_start];
|
|
|
|
|
|
|
|
// add BOS token for the first batch of each chunk
|
|
|
|
if (add_bos && j == 0) {
|
|
|
|
tokens[batch_start] = llama_token_bos(llama_get_model(ctx));
|
|
|
|
}
|
|
|
|
|
llama : greatly reduce output buffer memory usage (#6122)
* llama : greatly reduce logits memory usage
* llama : more compact state saving and reloading
* llama : fix lctx.n_outputs not being set before building graph
* perplexity : adapt to the logits API changes
* perplexity : fix Winogrande, use correct logits for second choice start
The first logits used to evaluate the second choice were not from
the end of the common prefix; instead, they were the logits from the end
of the first choice. This has been corrected.
The previous implementation sometimes had outliers in the scores of
choices for some tasks, and the logic to skip choices words
in the log-likelihood evaluation probably was an attempt to reduce those,
but it was complex and didn't quite seem to be the right thing.
This is simpler now, and the outlier scores aren't there anymore.
* perplexity : normalize spaces and punctuation in Winogrande sentences
* llama : fix embedding conditions
* llama : fix llama_get_embeddings_ith when the resulting id is 0
* llama : fix wrong n_outputs in llama_set_inputs
A mismatch happened when using a smaller n_ubatch than n_batch and then using
llama_batch_get_one(). The decision of what n_outputs should be now almost
fully depends on how lctx.n_outputs is set in llama_decode_internal.
The conditions are simpler this way.
* llama : when saving the state, recalculate n_outputs
This ensures the correct number of outputs for the entire previous batch
is stored in the session file, even when n_ubatch is smaller than n_batch.
* llama : fix not-skipping outputs of non-causal models
* llama : fix running a batch with n_outputs == 0
It previously worked because lctx.inp_out_ids was not initialized,
so it pointed to some garbage address which was somehow still valid when I
ran my tests.
* llama : keep same graph topology even when n_outputs == 0
* ggml : saner ggml_can_repeat with empty tensors
* ggml : future-proof ggml_is_empty by using GGML_MAX_DIMS - 1
* ggml : do not multi-thread ops returning empty tensors
* ggml : make ggml_is_empty public and work with views
* llama : use a vector for ctx->output_ids
* llama : rework reallocation logic for llama_output_reserve
Now comparing the actual size with the new total size of the output buffer
to allow more efficient enabling and disabling of the embeddings
and/or logits output in the future.
* ggml : skip empty tensors in all backends
* llama : fix llama_output_reserve nullptr deref when new_size is 0
* perplexity : make Winogrande work as it does on master
The problems with the Winogrande implementation will
need to be fixed in a separate PR to ease review.
* llama : clearer error messages for invalid logits or embeddings ids
* llama : assert all models that can have inp_out_ids
Since the graph topology is now constant, this presence check
can be done even when there are no outputs.
* llama : assert logits and embd buffers exist before writing to them
* llama : handle errors from llama_output_reserve at call sites
* perplexity : make hellaswag and multiple-choice outputs identical to master
Due to how the KV cache is updated, the logprobs for tokens in a batch
are very slightly affected by the other tokens present in the batch,
so to make hellaswag and multiple-choice return exactly the same results
as on master, the last token of each sequence needs to be evaluated
even though its output is not used at all.
This will probably be changed back in the future to make these benchmarks
a tiny bit faster.
* perplexity : fix division by zero when using less than 100 multiple-choice tasks
* llama : allow loading state saved with a different ctx size
When loading a session file, the context size is now only required to be
at least enough to load the KV cells contained in that session file,
instead of requiring to use exactly the same context size as when saving.
Doing this enables the use-case of extending or shrinking the context size
of a saved session.
This breaks existing session files because the meaning of kv_buf_size
is slightly changed (previously it was the size of the whole KV cache,
now it's only the size of the saved part of it). This allows for
finer-grained sanity checks when loading in an effort to keep kv_buf_size
useful even when the kv_size is changed.
* llama : minor
ggml-ci
* readme : update recent API changes, and warn about Vulkan
---------
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
2024-03-26 15:46:41 +01:00
|
|
|
// TODO: use batch.logits to save computations instead of relying on logits_all == true
|
2024-01-12 06:59:57 +01:00
|
|
|
if (llama_decode(ctx, llama_batch_get_one(tokens.data() + batch_start, batch_size, j * n_batch, 0))) {
|
|
|
|
fprintf(stderr, "%s : failed to eval\n", __func__);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
// restore the original token in case it was set to BOS
|
|
|
|
tokens[batch_start] = token_org;
|
|
|
|
|
2024-01-21 07:01:20 +01:00
|
|
|
if (compute_ppl && num_batches > 1) {
|
|
|
|
const auto * batch_logits = llama_get_logits(ctx);
|
|
|
|
logits.insert(logits.end(), batch_logits, batch_logits + batch_size * n_vocab);
|
|
|
|
}
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
const auto t_end = std::chrono::high_resolution_clock::now();
|
|
|
|
|
|
|
|
if (i == 0) {
|
|
|
|
const float t_total = std::chrono::duration<float>(t_end - t_start).count();
|
|
|
|
fprintf(stderr, "%s: %.2f seconds per pass - ETA ", __func__, t_total);
|
|
|
|
int total_seconds = (int)(t_total * n_chunk);
|
|
|
|
if (total_seconds >= 60*60) {
|
|
|
|
fprintf(stderr, "%d hours ", total_seconds / (60*60));
|
|
|
|
total_seconds = total_seconds % (60*60);
|
|
|
|
}
|
|
|
|
fprintf(stderr, "%.2f minutes\n", total_seconds / 60.0);
|
|
|
|
}
|
|
|
|
|
2024-01-21 07:01:20 +01:00
|
|
|
if (compute_ppl) {
|
|
|
|
const int first = n_ctx/2;
|
|
|
|
const auto all_logits = num_batches > 1 ? logits.data() : llama_get_logits(ctx);
|
|
|
|
process_logits(n_vocab, all_logits + first*n_vocab, tokens.data() + start + first, n_ctx - 1 - first,
|
|
|
|
workers, nll, nll2, logit_history.data() + start + first, prob_history.data() + start + first);
|
|
|
|
count += n_ctx - first - 1;
|
|
|
|
|
|
|
|
printf("[%d]%.4lf,", i + 1, std::exp(nll / count));
|
|
|
|
fflush(stdout);
|
2024-01-12 06:59:57 +01:00
|
|
|
|
2024-01-21 07:01:20 +01:00
|
|
|
logits.clear();
|
|
|
|
}
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
printf("\n");
|
|
|
|
|
2024-01-21 07:01:20 +01:00
|
|
|
if (compute_ppl) {
|
|
|
|
nll2 /= count;
|
|
|
|
nll /= count;
|
|
|
|
const double ppl = exp(nll);
|
|
|
|
nll2 -= nll * nll;
|
|
|
|
if (nll2 > 0) {
|
|
|
|
nll2 = sqrt(nll2/(count-1));
|
|
|
|
printf("Final estimate: PPL = %.4lf +/- %.5lf\n", ppl, nll2*ppl);
|
|
|
|
} else {
|
|
|
|
printf("Unexpected negative standard deviation of log(prob)\n");
|
|
|
|
}
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
int main(int argc, char ** argv) {
|
|
|
|
|
|
|
|
StatParams sparams;
|
2024-02-04 09:39:58 +01:00
|
|
|
std::string prev_result_file;
|
|
|
|
std::string combine_files;
|
2024-01-21 07:01:20 +01:00
|
|
|
bool compute_ppl = true;
|
2024-02-04 09:39:58 +01:00
|
|
|
int from_chunk = 0;
|
2024-01-12 06:59:57 +01:00
|
|
|
std::vector<char*> args;
|
|
|
|
args.push_back(argv[0]);
|
|
|
|
int iarg = 1;
|
|
|
|
for (; iarg < argc-1; ++iarg) {
|
|
|
|
std::string arg{argv[iarg]};
|
|
|
|
if (arg == "-o" || arg == "--output-file") {
|
|
|
|
sparams.ofile = argv[++iarg];
|
|
|
|
}
|
|
|
|
else if (arg == "-ofreq" || arg == "--output-frequency") {
|
|
|
|
sparams.n_output_frequency = std::stoi(argv[++iarg]);
|
|
|
|
}
|
|
|
|
else if (arg == "-ow" || arg == "--output-weight") {
|
|
|
|
sparams.collect_output_weight = std::stoi(argv[++iarg]);
|
|
|
|
}
|
|
|
|
else if (arg == "--verbosity") {
|
|
|
|
sparams.verbosity = std::stoi(argv[++iarg]);
|
2024-01-21 07:01:20 +01:00
|
|
|
} else if (arg == "--no-ppl") {
|
|
|
|
compute_ppl = false;
|
2024-01-22 13:18:43 +01:00
|
|
|
} else if (arg == "--keep-imatrix") {
|
|
|
|
sparams.keep_every = std::stoi(argv[++iarg]);
|
2024-02-04 09:39:58 +01:00
|
|
|
} else if (arg == "--continue-from") {
|
|
|
|
prev_result_file = argv[++iarg];
|
|
|
|
} else if (arg == "--combine") {
|
|
|
|
combine_files = argv[++iarg];
|
|
|
|
}
|
|
|
|
else if (arg == "--from-chunk") {
|
|
|
|
from_chunk = std::stoi(argv[++iarg]);
|
2024-01-12 06:59:57 +01:00
|
|
|
} else {
|
|
|
|
args.push_back(argv[iarg]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (iarg < argc) {
|
2024-01-21 07:01:20 +01:00
|
|
|
std::string arg{argv[iarg]};
|
|
|
|
if (arg == "--no-ppl") {
|
|
|
|
compute_ppl = false;
|
|
|
|
} else {
|
|
|
|
args.push_back(argv[iarg]);
|
|
|
|
}
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
|
2024-04-26 20:06:33 +02:00
|
|
|
gpt_params params;
|
|
|
|
params.n_batch = 512;
|
|
|
|
if (!gpt_params_parse(args.size(), args.data(), params)) {
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
params.logits_all = true;
|
|
|
|
params.n_batch = std::min(params.n_batch, params.n_ctx);
|
|
|
|
|
|
|
|
print_build_info();
|
|
|
|
|
|
|
|
if (params.seed == LLAMA_DEFAULT_SEED) {
|
|
|
|
params.seed = time(NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
fprintf(stderr, "%s: seed = %u\n", __func__, params.seed);
|
|
|
|
|
|
|
|
std::mt19937 rng(params.seed);
|
|
|
|
if (params.random_prompt) {
|
2024-05-22 19:04:20 +02:00
|
|
|
params.prompt = string_random_prompt(rng);
|
2024-04-26 20:06:33 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
sparams.dataset = params.prompt_file;
|
2024-02-04 09:39:58 +01:00
|
|
|
g_collector.set_parameters(std::move(sparams));
|
|
|
|
|
|
|
|
if (!combine_files.empty()) {
|
|
|
|
std::vector<std::string> files;
|
|
|
|
size_t pos = 0;
|
|
|
|
while (true) {
|
|
|
|
auto new_pos = combine_files.find(',', pos);
|
|
|
|
if (new_pos != std::string::npos) {
|
|
|
|
files.emplace_back(combine_files.substr(pos, new_pos - pos));
|
|
|
|
pos = new_pos + 1;
|
|
|
|
} else {
|
|
|
|
files.emplace_back(combine_files.substr(pos));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (files.size() < 2) {
|
|
|
|
fprintf(stderr, "You must provide at least two comma separated files to use --combine\n");
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
printf("Combining the following %d files\n", int(files.size()));
|
|
|
|
for (auto& file : files) {
|
|
|
|
printf(" %s\n", file.c_str());
|
|
|
|
if (!g_collector.load_imatrix(file.c_str(), true)) {
|
|
|
|
fprintf(stderr, "Failed to load %s\n", file.c_str());
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
g_collector.save_imatrix();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!prev_result_file.empty()) {
|
|
|
|
if (!g_collector.load_imatrix(prev_result_file.c_str(), false)) {
|
|
|
|
fprintf(stderr, "=============== Failed to load %s\n", prev_result_file.c_str());
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-16 10:31:07 +01:00
|
|
|
llama_backend_init();
|
|
|
|
llama_numa_init(params.numa);
|
2024-01-12 06:59:57 +01:00
|
|
|
|
2024-01-17 17:46:30 +01:00
|
|
|
// pass the callback to the backend scheduler
|
|
|
|
// it will be executed for each node during the graph computation
|
2024-04-11 14:51:07 +02:00
|
|
|
params.cb_eval = ik_collect_imatrix;
|
|
|
|
params.cb_eval_user_data = NULL;
|
|
|
|
params.warmup = false;
|
|
|
|
|
|
|
|
// init
|
|
|
|
llama_model * model;
|
|
|
|
llama_context * ctx;
|
|
|
|
std::tie(model, ctx) = llama_init_from_gpt_params(params);
|
|
|
|
if (model == nullptr || ctx == nullptr) {
|
|
|
|
fprintf(stderr, "%s : failed to init\n", __func__);
|
2024-01-17 17:46:30 +01:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2024-01-12 06:59:57 +01:00
|
|
|
const int n_ctx_train = llama_n_ctx_train(model);
|
|
|
|
if (params.n_ctx > n_ctx_train) {
|
|
|
|
fprintf(stderr, "%s: warning: model was trained on only %d context tokens (%d specified)\n",
|
|
|
|
__func__, n_ctx_train, params.n_ctx);
|
|
|
|
}
|
|
|
|
|
|
|
|
// print system information
|
|
|
|
{
|
|
|
|
fprintf(stderr, "\n");
|
2024-05-22 19:04:20 +02:00
|
|
|
fprintf(stderr, "%s\n", gpt_params_get_system_info(params).c_str());
|
2024-01-12 06:59:57 +01:00
|
|
|
}
|
|
|
|
|
2024-02-04 09:39:58 +01:00
|
|
|
bool OK = compute_imatrix(ctx, params, compute_ppl, from_chunk);
|
2024-01-12 06:59:57 +01:00
|
|
|
if (!OK) {
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
g_collector.save_imatrix();
|
|
|
|
|
|
|
|
llama_print_timings(ctx);
|
|
|
|
|
|
|
|
llama_free(ctx);
|
|
|
|
llama_free_model(model);
|
|
|
|
|
|
|
|
llama_backend_free();
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|