gguf : single pass for writing tensors + refactoring writer

convert-llama-h5-to-gguf.py

@@ -18,12 +18,16 @@ NDArray: 'TypeAlias' = 'np.ndarray[Any, Any]'
 
 # reverse HF permute back to original pth layout
 # https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/convert_llama_weights_to_hf.py
+
+
 def reverse_hf_permute(weights: NDArray, n_head: int, n_kv_head: Optional[int] = None) -> NDArray:
-    if n_kv_head is not None and n_head != n_kv_head: n_head //= n_kv_head
+    if n_kv_head is not None and n_head != n_kv_head:
+        n_head //= n_kv_head
     return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])
             .swapaxes(1, 2)
             .reshape(weights.shape))
 
+
 def count_model_parts(dir_model: str) -> int:
     num_parts = 0
     for filename in os.listdir(dir_model):
@@ -34,6 +38,7 @@ def count_model_parts(dir_model: str) -> int:
         print("gguf: found " + str(num_parts) + " model parts")
     return num_parts
 
+
 if len(sys.argv) < 3:
     print("Usage: convert-h5-to-ggml.py dir-model ftype\n")
     print("  ftype == 0 -> float32")
@@ -74,12 +79,11 @@ if hparams["architectures"][0] != "LlamaForCausalLM":
 # get number of model parts
 num_parts = count_model_parts(dir_model)
 
-gguf_writer = gguf.GGUFWriter.open(fname_out)
+gguf_writer = gguf.GGUFWriter(fname_out, architecture="llama")
 
 
 print("gguf: get model metadata")
 
-llm_arch = "llama"
 block_count = hparams["num_hidden_layers"]
 head_count = hparams["num_attention_heads"]
 
@@ -91,7 +95,7 @@ else:
 if "_name_or_path" in hparams:
     hf_repo = hparams["_name_or_path"]
 else:
-    hf_repo=""
+    hf_repo = ""
 
 if "max_sequence_length" in hparams:
     ctx_length = hparams["max_sequence_length"]
@@ -102,19 +106,19 @@ else:
     sys.exit()
 
 
-gguf_writer.add_architecture(llm_arch)
+gguf_writer.add_architecture()
 gguf_writer.add_name(last_dir)
 gguf_writer.add_file_type("All tensors F32" if ftype == 0 else "Most tensors F16, some F32")
 gguf_writer.add_source_hf_repo(hf_repo)
-gguf_writer.add_tensor_data_layout(llm_arch, "Meta AI original pth")
+gguf_writer.add_tensor_data_layout("Meta AI original pth")
-gguf_writer.add_context_length(llm_arch, ctx_length)
+gguf_writer.add_context_length(ctx_length)
-gguf_writer.add_embedding_length(llm_arch, hparams["hidden_size"])
+gguf_writer.add_embedding_length(hparams["hidden_size"])
-gguf_writer.add_block_count(llm_arch, block_count)
+gguf_writer.add_block_count(block_count)
-gguf_writer.add_feed_forward_length(llm_arch, hparams["intermediate_size"])
+gguf_writer.add_feed_forward_length(hparams["intermediate_size"])
-gguf_writer.add_rope_dimension_count(llm_arch, hparams["hidden_size"] // hparams["num_attention_heads"])
+gguf_writer.add_rope_dimension_count(hparams["hidden_size"] // hparams["num_attention_heads"])
-gguf_writer.add_head_count(llm_arch, head_count)
+gguf_writer.add_head_count(head_count)
-gguf_writer.add_head_count_kv(llm_arch, head_count_kv)
+gguf_writer.add_head_count_kv(head_count_kv)
-gguf_writer.add_layer_norm_rms_eps(llm_arch, hparams["rms_norm_eps"])
+gguf_writer.add_layer_norm_rms_eps(hparams["rms_norm_eps"])
 
 
 # TOKENIZATION
@@ -140,15 +144,19 @@ if Path(dir_model + "/tokenizer.model").is_file():
         score = tokenizer.get_score(i)
 
         toktype = 1  # defualt to normal token type
-        if tokenizer.is_unknown(i): toktype = 2
+        if tokenizer.is_unknown(i):
-        if tokenizer.is_control(i): toktype = 3
+            toktype = 2
+        if tokenizer.is_control(i):
+            toktype = 3
 
         # TODO: How to determinate if a token is user defined?
         # ref: https://github.com/google/sentencepiece/blob/master/src/sentencepiece_model.proto
         # if tokenizer.is_user_defined(i): toktype = 4
 
-        if tokenizer.is_unused(i):  toktype = 5
+        if tokenizer.is_unused(i):
-        if tokenizer.is_byte(i):    toktype = 6
+            toktype = 5
+        if tokenizer.is_byte(i):
+            toktype = 6
 
         tokens.append(text)
         scores.append(score)
@@ -212,7 +220,7 @@ else:
     )
 
 for part_name in part_names:
-    print("gguf: loading model part '"+ part_name + "'")
+    print("gguf: loading model part '" + part_name + "'")
     model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu")
 
     for name in model_part.keys():
@@ -238,11 +246,12 @@ for part_name in part_names:
         elif name.endswith(".bias") and name[:-5] in tensor_map:
             name = tensor_map[name[:-5]] + ".bias"
         else:
-            print( "Can not map tensor '" + name + "'" )
+            print("Can not map tensor '" + name + "'")
             sys.exit()
 
         n_dims = len(data.shape)
         data_dtype = data.dtype
+        old_dtype = data_dtype
 
         # if f32 desired, convert any float16 to float32
         if ftype == 0 and data.dtype == np.float16:
@@ -256,17 +265,19 @@ for part_name in part_names:
         if ftype == 1 and data.dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
             data_dtype = np.float16
 
-        data_nbytes = data.size * 2 if data_dtype == np.float16 else data.size * 4
+        data = data.astype(data_dtype)
 
-        gguf_writer.add_tensor_info(name, data.shape, data_dtype, data_nbytes)
+        print(name + ", n_dims = " + n_dims + ", " + str(old_dtype) + " --> " + str(data.dtype))
+
+        gguf_writer.add_tensor(name, data)
 
 
 print("gguf: write header")
 gguf_writer.write_header_to_file()
 print("gguf: write metadata")
 gguf_writer.write_kv_data_to_file()
-print("gguf: write tensor metadata")
+print("gguf: write tensors")
-gguf_writer.write_ti_data_to_file()
+gguf_writer.write_tensors_to_file()
 
 # tensor data
 print("gguf: convert and write tensor data")
@@ -279,7 +290,7 @@ else:
     )
 
 for part_name in part_names:
-    print("gguf: loading model part '"+ part_name + "'")
+    print("gguf: loading model part '" + part_name + "'")
     model_part = torch.load(f"{dir_model}/{part_name}", map_location="cpu")
 
     for name in model_part.keys():
@@ -307,7 +318,7 @@ for part_name in part_names:
         elif name.endswith(".bias") and name[:-5] in tensor_map:
             name = tensor_map[name[:-5]] + ".bias"
         else:
-            print( "Can not map tensor '" + name + "'" )
+            print("Can not map tensor '" + name + "'")
             sys.exit()
 
         n_dims = len(data.shape)
@@ -325,8 +336,6 @@ for part_name in part_names:
         if ftype == 1 and data_dtype == np.float32 and name.endswith(".weight") and n_dims == 2:
             data = data.astype(np.float16)
 
-        print(name + ", shape " + str(len(data.shape)) + ", " + str(old_dtype) + " --> " + str(data.dtype))
-
         gguf_writer.write_tensor_to_file(data)
 
 gguf_writer.close()

gguf.py

@@ -1,11 +1,7 @@
-"""TODOs
+import shutil
-1. Implement writers for known architectures, LLaMA in particular.
-2. Add docstrings from the format specs.
-3. After development is done, Convert it to a proper pip-installable Python package, and possibly move it to its own repo under ggml-org.
-"""
-
 import sys
 import struct
+import tempfile
 import numpy as np
 
 from enum import IntEnum
@@ -70,7 +66,8 @@ KEY_TOKENIZER_RWKV       = "tokenizer.rwkv.world"
 # recommended mapping of model tensor names for storage in gguf
 #
 
-def get_tensor_name_map(n_blocks : int):
+
+def get_tensor_name_map(n_blocks: int):
     tensor_map = {}
     # Token embeddings
     mapped_to = "token_embd"
@@ -95,7 +92,7 @@ def get_tensor_name_map(n_blocks : int):
     tensor_map["lm_head"] = mapped_to  # gpt2 mpt falcon llama-hf
     tensor_map["output"] = mapped_to  # llama-pth
     # Attention and fee-forward layer blocks
-    for i in range(0,n_blocks):
+    for i in range(0, n_blocks):
         # Attention norm
         mapped_to = "blk."+str(i)+".attn_norm"
         tensor_map["gpt_neox.layers."+str(i)+".input_layernorm"] = mapped_to  # gptneox
@@ -168,6 +165,7 @@ def get_tensor_name_map(n_blocks : int):
 # implementation
 #
 
+
 class GGMLQuantizationType(IntEnum):
     F32 = 0
     F16 = 1
@@ -203,8 +201,9 @@ class GGUFValueType(IntEnum):
 
 
 class GGUFWriter:
-    def __init__(self, fout: IO):
+    def __init__(self, path: str, architecture: str):
-        self.fout = fout
+        self.fout = open(path, "wb")
+        self.arch = architecture
         self.offset_tensor = 0
         self.data_alignment = GGUF_DEFAULT_ALIGNMENT
         self.kv_data = b""
@@ -228,11 +227,6 @@ class GGUFWriter:
         self.fout.write(self.ti_data)
         self.flush()
 
-    @classmethod
-    def open(cls, path: str) -> "GGUFWriter":
-        f = open(path, "wb")
-        return cls(f)
-
     def add_key(self, key: str):
         self.add_val(key, GGUFValueType.STRING, add_vtype=False)
 
@@ -269,7 +263,8 @@ class GGUFWriter:
         self.add_val(val, GGUFValueType.BOOL)
 
     def add_string(self, key: str, val: str):
-        if len(val) == 0: return
+        if len(val) == 0:
+            return
         self.add_key(key)
         self.add_val(val, GGUFValueType.STRING)
 
@@ -323,6 +318,8 @@ class GGUFWriter:
         return ((x + n - 1) // n) * n
 
     def add_tensor_info(self, name: str, tensor_shape: np.ndarray, tensor_dtype: np.dtype, tensor_nbytes: int):
+        assert tensor_dtype in (np.float32, np.float16), "Only F32 and F16 tensors are supported for now"
+
         encoded_name = name.encode("utf8")
         self.ti_data += struct.pack("<I", len(encoded_name))
         self.ti_data += encoded_name
@@ -331,13 +328,25 @@ class GGUFWriter:
         for i in range(n_dims):
             self.ti_data += struct.pack("<I", tensor_shape[n_dims - 1 - i])
 
-        assert tensor_dtype in (np.float32, np.float16), "Only F32 and F16 tensors are supported for now"
         dtype = GGMLQuantizationType.F32 if tensor_dtype == np.float32 else GGMLQuantizationType.F16
         self.ti_data += struct.pack("<I", dtype)
         self.ti_data += struct.pack("<Q", self.offset_tensor)
         self.offset_tensor += GGUFWriter.ggml_pad(tensor_nbytes, self.data_alignment)
         self.ti_data_count += 1
 
+    def add_tensor(self, name: str, tensor: np.ndarray):
+        if not hasattr(self, "temp_file"):
+            self.temp_file = tempfile.SpooledTemporaryFile(mode="w+b", max_size=256*1024*1024)
+            self.temp_file.seek(0)
+
+        self.add_tensor_info(name, tensor.shape, tensor.dtype, tensor.nbytes)
+
+        tensor.tofile(self.temp_file)
+
+        pad = GGUFWriter.ggml_pad(tensor.nbytes, self.data_alignment) - tensor.nbytes
+        if pad != 0:
+            self.temp_file.write(bytes([0] * pad))
+
     def write_tensor_to_file(self, tensor: np.ndarray):
         pad = GGUFWriter.ggml_pad(self.fout.tell(), self.data_alignment) - self.fout.tell()
         if pad != 0:
@@ -349,21 +358,33 @@ class GGUFWriter:
         if pad != 0:
             self.fout.write(bytes([0] * pad))
 
+    def write_tensors_to_file(self):
+        self.write_ti_data_to_file()
+
+        pad = GGUFWriter.ggml_pad(self.fout.tell(), self.data_alignment) - self.fout.tell()
+        if pad != 0:
+            self.fout.write(bytes([0] * pad))
+
+        self.temp_file.seek(0)
+
+        shutil.copyfileobj(self.temp_file, self.fout)
+        self.flush()
+        self.temp_file.close()
+
     def flush(self):
         self.fout.flush()
 
     def close(self):
         self.fout.close()
 
-    def add_architecture(self, architecture: str):
+    def add_architecture(self):
-        self.add_string(KEY_GENERAL_ARCHITECTURE,
+        self.add_string(KEY_GENERAL_ARCHITECTURE, self.arch)
-                        architecture)
 
     def add_author(self, author: str):
         self.add_string(KEY_GENERAL_AUTHOR, author)
 
     def add_tensor_data_layout(self, layout: str):
-        self.add_string(KEY_LLM_TENSOR_DATA_LAYOUT , layout)
+        self.add_string(KEY_LLM_TENSOR_DATA_LAYOUT.format(llm=self.arch), layout)
 
     def add_url(self, url: str):
         self.add_string(KEY_GENERAL_URL, url)
@@ -391,60 +412,60 @@ class GGUFWriter:
         self.data_alignment = alignment
         self.add_uint32(KEY_GENERAL_ALIGNMENT, alignment)
 
-    def add_context_length(self, llm: str, length: int):
+    def add_context_length(self, length: int):
         self.add_uint32(
-            KEY_LLM_CONTEXT_LENGTH.format(llm=llm), length)
+            KEY_LLM_CONTEXT_LENGTH.format(llm=self.arch), length)
 
-    def add_embedding_length(self, llm: str, length: int):
+    def add_embedding_length(self, length: int):
         self.add_uint32(
-            KEY_LLM_EMBEDDING_LENGTH.format(llm=llm), length)
+            KEY_LLM_EMBEDDING_LENGTH.format(llm=self.arch), length)
 
-    def add_block_count(self, llm: str, length: int):
+    def add_block_count(self, length: int):
         self.add_uint32(
-            KEY_LLM_BLOCK_COUNT.format(llm=llm), length)
+            KEY_LLM_BLOCK_COUNT.format(llm=self.arch), length)
 
-    def add_feed_forward_length(self, llm: str, length: int):
+    def add_feed_forward_length(self, length: int):
         self.add_uint32(
-            KEY_LLM_FEED_FORWARD_LENGTH.format(llm=llm), length)
+            KEY_LLM_FEED_FORWARD_LENGTH.format(llm=self.arch), length)
 
-    def add_parallel_residual(self, llm: str, use: bool):
+    def add_parallel_residual(self, use: bool):
         self.add_bool(
-            KEY_LLM_USE_PARALLEL_RESIDUAL.format(llm=llm), use)
+            KEY_LLM_USE_PARALLEL_RESIDUAL.format(llm=self.arch), use)
 
-    def add_tensor_data_layout(self, llm: str, layout: str):
+    def add_tensor_data_layout(self, layout: str):
         self.add_string(
-            KEY_LLM_TENSOR_DATA_LAYOUT.format(llm=llm), layout)
+            KEY_LLM_TENSOR_DATA_LAYOUT.format(llm=self.arch), layout)
 
-    def add_head_count(self, llm: str, count: int):
+    def add_head_count(self, count: int):
         self.add_uint32(
-            KEY_ATTENTION_HEAD_COUNT.format(llm=llm), count)
+            KEY_ATTENTION_HEAD_COUNT.format(llm=self.arch), count)
 
-    def add_head_count_kv(self, llm: str, count: int):
+    def add_head_count_kv(self, count: int):
         self.add_uint32(
-            KEY_ATTENTION_HEAD_COUNT_KV.format(llm=llm), count)
+            KEY_ATTENTION_HEAD_COUNT_KV.format(llm=self.arch), count)
 
-    def add_max_alibi_bias(self, llm: str, bias: float):
+    def add_max_alibi_bias(self, bias: float):
         self.add_float32(
-            KEY_ATTENTION_MAX_ALIBI_BIAS.format(llm=llm), bias)
+            KEY_ATTENTION_MAX_ALIBI_BIAS.format(llm=self.arch), bias)
 
-    def add_clamp_kqv(self, llm: str, value: float):
+    def add_clamp_kqv(self, value: float):
         self.add_float32(
-            KEY_ATTENTION_CLAMP_KQV.format(llm=llm), value)
+            KEY_ATTENTION_CLAMP_KQV.format(llm=self.arch), value)
 
-    def add_layer_norm_eps(self, llm: str, value: float):
+    def add_layer_norm_eps(self, value: float):
         self.add_float32(
-            KEY_ATTENTION_LAYERNORM_EPS.format(llm=llm), value)
+            KEY_ATTENTION_LAYERNORM_EPS.format(llm=self.arch), value)
 
-    def add_layer_norm_rms_eps(self, llm: str, value: float):
+    def add_layer_norm_rms_eps(self, value: float):
         self.add_float32(
-            KEY_ATTENTION_LAYERNORM_RMS_EPS.format(llm=llm), value)
+            KEY_ATTENTION_LAYERNORM_RMS_EPS.format(llm=self.arch), value)
 
-    def add_rope_dimension_count(self, llm: str, count: int):
+    def add_rope_dimension_count(self, count: int):
         self.add_uint32(
-            KEY_ROPE_DIMENSION_COUNT.format(llm=llm), count)
+            KEY_ROPE_DIMENSION_COUNT.format(llm=self.arch), count)
 
-    def add_rope_scale(self, llm: str, value:  float):
+    def add_rope_scale(self, value:  float):
-        self.add_float32(KEY_ROPE_SCALE.format(llm=llm), value)
+        self.add_float32(KEY_ROPE_SCALE.format(llm=self.arch), value)
 
     def add_tokenizer_model(self, model: str):
         self.add_string(KEY_TOKENIZER_MODEL, model)
@@ -476,24 +497,27 @@ class GGUFWriter:
     def add_pad_token_id(self, id: int):
         self.add_uint32(KEY_TOKENIZER_PAD_ID, id)
 
+
 # Example usage:
 if __name__ == "__main__":
     # Example usage with a file
-    gguf_writer = GGUFWriter.open("example.gguf")
+    gguf_writer = GGUFWriter("example.gguf", "llama")
 
-    gguf_writer.add_architecture("llama")
+    gguf_writer.add_architecture()
+    gguf_writer.add_block_count(12)
     gguf_writer.add_uint32("answer", 42)  # Write a 32-bit integer
     gguf_writer.add_float32("answer_in_float", 42.0)  # Write a 32-bit float
     gguf_writer.add_custom_alignment(64)
     tensor1 = np.ones((32,), dtype=np.float32) * 100.0
-    tensor2 = np.ones((32,), dtype=np.float32) * 101.0
+    tensor2 = np.ones((64,), dtype=np.float32) * 101.0
-    gguf_writer.add_tensor_info("tensor0", tensor1)
+    tensor3 = np.ones((96,), dtype=np.float32) * 102.0
-    gguf_writer.add_tensor_info("tensor1", tensor2)
+
+    gguf_writer.add_tensor("tensor1", tensor1)
+    gguf_writer.add_tensor("tensor2", tensor2)
+    gguf_writer.add_tensor("tensor3", tensor3)
 
     gguf_writer.write_header_to_file()
     gguf_writer.write_kv_data_to_file()
-    gguf_writer.write_ti_data_to_file()
+    gguf_writer.write_tensors_to_file()
-    gguf_writer.write_tensor_to_file(tensor1)
-    gguf_writer.write_tensor_to_file(tensor2)
 
     gguf_writer.close()