From 278ada9572c645a8353edce256453a755ed1743e Mon Sep 17 00:00:00 2001 From: klosax <131523366+klosax@users.noreply.github.com> Date: Fri, 4 Aug 2023 04:07:57 +0200 Subject: [PATCH] gguf.py : bytesarray for gpt2bpe tokenizer --- gguf.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gguf.py b/gguf.py index 88e2bee07..1e4f8ea3b 100644 --- a/gguf.py +++ b/gguf.py @@ -10,7 +10,7 @@ from enum import IntEnum from typing import Any, IO, List import numpy as np - +import sys class GGMLQuantizationType(IntEnum): F32 = 0 @@ -45,7 +45,7 @@ class GGUFValueType(IntEnum): @staticmethod def get_type(val): - if isinstance(val, str) or isinstance(val, bytes): + if isinstance(val, str) or isinstance(val, bytes) or isinstance(val, bytearray): return GGUFValueType.STRING elif isinstance(val, list): return GGUFValueType.ARRAY @@ -53,8 +53,11 @@ class GGUFValueType(IntEnum): return GGUFValueType.FLOAT32 elif isinstance(val, bool): return GGUFValueType.BOOL - else: + elif isinstance(val, int): return GGUFValueType.INT32 + else: + print("Unknown type: "+str(type(val))) + sys.exit() class GGUFWriter: