Mirrors/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-07 11:23:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

convert : write more metadata for LLaMA

Browse Source
This commit is contained in:
M. Yusuf Sarıgöz 2023-08-13 13:29:46 +03:00
parent 9bf5a7efcb
commit 91d4bfd536
2 changed files with 20 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
convert-llama-h5-to-gguf.py
View File

@ -17,6 +17,7 @@ from sentencepiece import SentencePieceProcessor
# compatible with python < 3.9
NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]'
def permute(weights: NDArray, n_head: int) -> NDArray:
return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])
.swapaxes(1, 2)
@ -69,17 +70,22 @@ gguf_writer = gguf.GGUFWriter.open(fname_out)
print("gguf: get model metadata")
llm_arch = "llama"
hf_repo = hparams["_name_or_path"]
head_count = hparams["num_attention_heads"]
head_count_kv = hparams["num_key_value_heads"]
block_count = hparams["num_hidden_layers"]
gguf_writer.add_name(last_dir)
gguf_writer.add_architecture(llm_arch)
gguf_writer.add_quantization_version(ftype)
guff_writer.add_source_hf_repo(hf_repo)
gguf_writer.add_context_length(llm_arch, hparams["max_position_embeddings"])
gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"])
gguf_writer.add_block_count(llm_arch, block_count)
gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"])
gguf_writer.add_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"])
gguf_writer.add_head_count(llm_arch, head_count)
gguf_writer.add_head_count_kv(llm_arch, head_count_kv)
gguf_writer.add_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"])

17
gguf.py
View File

@ -12,23 +12,10 @@ from typing import Any, IO, List
import numpy as np
import sys
class GGMLQuantizationType(IntEnum):
F32 = 0
F16 = 1
Q4_0 = 2
Q4_1 = 3
# Q4_2 = 4 # support has been removed
# Q4_3 = 5 # support has been removed
Q5_0 = 6
Q5_1 = 7
Q8_0 = 8
Q8_1 = 9
Q2_K = 10
Q3_K = 11
Q4_K = 12
Q5_K = 13
Q6_K = 14
Q8_K = 15
class GGUFValueType(IntEnum):
@ -143,7 +130,7 @@ class GGUFWriter:
if add_vtype:
self.kv_data += struct.pack("<I", vtype)
self.kv_data_count += 1;
self.kv_data_count += 1
if vtype == GGUFValueType.UINT8:
self.kv_data += struct.pack("<B", val)