Mirrors/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-25 13:58:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

Change RMSNorm eps to 1e-6 (#173)

Browse Source 
I think this is what is used in the Python code
This commit is contained in:
Georgi Gerganov 2023-03-19 17:30:00 +02:00
parent d7def1a752
commit 22213a17b5
No known key found for this signature in database
GPG Key ID: 449E073F9DC10735
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ggml.c
View File

@ -5556,7 +5556,7 @@ static void ggml_compute_forward_rms_norm_f32(
const size_t nb2 = dst->nb[2]; const size_t nb2 = dst->nb[2];
const size_t nb3 = dst->nb[3]; const size_t nb3 = dst->nb[3];
const ggml_float eps = 1e-5f; // TODO: make this a parameter const ggml_float eps = 1e-6f; // TODO: make this a parameter
// TODO: optimize // TODO: optimize
for (int i03 = 0; i03 < ne03; i03++) { for (int i03 = 0; i03 < ne03; i03++) {
@ -5572,7 +5572,7 @@ static void ggml_compute_forward_rms_norm_f32(
mean /= ne00; mean /= ne00;
float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3); float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
memcpy(y, x, ne00 * sizeof(float)); memcpy(y, x, ne00 * sizeof(float));
// for (int i00 = 0; i00 < ne00; i00++) { // for (int i00 = 0; i00 < ne00; i00++) {
// y[i00] = x[i00]; // y[i00] = x[i00];