From dc685be46622a8fabfd57cfa804237c8f15679b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sun, 12 May 2024 19:40:45 +0200 Subject: [PATCH 001/137] CUDA: add FP32 FlashAttention vector kernel (#7188) * CUDA: add FP32 FlashAttention vector kernel * fixup! CUDA: add FP32 FlashAttention vector kernel * fixup! fixup! CUDA: add FP32 FlashAttention vector kernel * fixup! fixup! fixup! CUDA: add FP32 FlashAttention vector kernel --- ggml-cuda.cu | 11 +- ggml-cuda/common.cuh | 4 + ggml-cuda/fattn-common.cuh | 47 ++++ ggml-cuda/fattn-vec-f16.cu | 430 +++++++++++++++++++++++++++++++++ ggml-cuda/fattn-vec-f16.cuh | 5 + ggml-cuda/fattn-vec-f32.cu | 384 +++++++++++++++++++++++++++++ ggml-cuda/fattn-vec-f32.cuh | 3 + ggml-cuda/fattn.cu | 468 ++---------------------------------- tests/test-backend-ops.cpp | 5 +- 9 files changed, 899 insertions(+), 458 deletions(-) create mode 100644 ggml-cuda/fattn-common.cuh create mode 100644 ggml-cuda/fattn-vec-f16.cu create mode 100644 ggml-cuda/fattn-vec-f16.cuh create mode 100644 ggml-cuda/fattn-vec-f32.cu create mode 100644 ggml-cuda/fattn-vec-f32.cuh diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 5b6c90919..75a2ad480 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -2713,6 +2713,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t } GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tensor * op) { + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; switch (op->op) { case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { @@ -2840,8 +2841,16 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons case GGML_OP_ARANGE: case GGML_OP_TIMESTEP_EMBEDDING: case GGML_OP_LEAKY_RELU: - case GGML_OP_FLASH_ATTN_EXT: return true; + case GGML_OP_FLASH_ATTN_EXT: +#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) + return op->src[0]->ne[0] == 64 || op->src[0]->ne[0] == 128; +#else + if (op->src[0]->ne[0] == 64 || op->src[0]->ne[0] == 128) { + return true; + } + return ggml_cuda_info().devices[cuda_ctx->device].cc >= CC_VOLTA; +#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) default: return false; } diff --git a/ggml-cuda/common.cuh b/ggml-cuda/common.cuh index 44e67e040..b6f0bc36a 100644 --- a/ggml-cuda/common.cuh +++ b/ggml-cuda/common.cuh @@ -321,6 +321,10 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) { #define FP16_MMA_AVAILABLE !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) && __CUDA_ARCH__ >= CC_VOLTA +static bool fast_fp16_available(const int cc) { + return cc >= CC_PASCAL && cc != 610; +} + static bool fp16_mma_available(const int cc) { return cc < CC_OFFSET_AMD && cc >= CC_VOLTA; } diff --git a/ggml-cuda/fattn-common.cuh b/ggml-cuda/fattn-common.cuh new file mode 100644 index 000000000..33f640691 --- /dev/null +++ b/ggml-cuda/fattn-common.cuh @@ -0,0 +1,47 @@ +#define FATTN_KQ_STRIDE 256 +#define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction. +#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs. + +template // D == head size +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +__launch_bounds__(D, 1) +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +static __global__ void flash_attn_combine_results( + const float * __restrict__ VKQ_parts, + const float2 * __restrict__ VKQ_meta, + float * __restrict__ dst) { + VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x; + VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x; + dst += D * gridDim.y*blockIdx.x; + + const int tid = threadIdx.x; + __builtin_assume(tid < D); + + __shared__ float2 meta[parallel_blocks]; + if (tid < 2*parallel_blocks) { + ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid]; + } + + __syncthreads(); + + float kqmax = meta[0].x; +#pragma unroll + for (int l = 1; l < parallel_blocks; ++l) { + kqmax = max(kqmax, meta[l].x); + } + + float VKQ_numerator = 0.0f; + float VKQ_denominator = 0.0f; +#pragma unroll + for (int l = 0; l < parallel_blocks; ++l) { + const float diff = meta[l].x - kqmax; + const float KQ_max_scale = expf(diff); + const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD); + *((uint32_t *) &KQ_max_scale) &= ftz_mask; + + VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid]; + VKQ_denominator += KQ_max_scale * meta[l].y; + } + + dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator; +} diff --git a/ggml-cuda/fattn-vec-f16.cu b/ggml-cuda/fattn-vec-f16.cu new file mode 100644 index 000000000..cbf5f7835 --- /dev/null +++ b/ggml-cuda/fattn-vec-f16.cu @@ -0,0 +1,430 @@ +#include "common.cuh" +#include "fattn-common.cuh" +#include "fattn-vec-f16.cuh" + +template // D == head size +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +__launch_bounds__(D, 1) +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +static __global__ void flash_attn_vec_ext_f16( + const char * __restrict__ Q, + const char * __restrict__ K, + const char * __restrict__ V, + const char * __restrict__ mask, + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, + const float max_bias, + const float m0, + const float m1, + const uint32_t n_head_log2, + const int ne00, + const int ne01, + const int ne02, + const int ne03, + const int ne10, + const int ne11, + const int ne12, + const int ne13, + const int ne31, + const int nb31, + const int nb01, + const int nb02, + const int nb03, + const int nb11, + const int nb12, + const int nb13, + const int ne0, + const int ne1, + const int ne2, + const int ne3) { +#if FP16_AVAILABLE + //In this kernel Q, K, V are matrices while i, j, k are matrix indices. + + const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. + const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + + const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. + const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); + const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); + const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const half * maskh = (const half *) mask + ne11*ic0; + + const int stride_KV = nb11 / sizeof(half); + const int stride_KV2 = nb11 / sizeof(half2); + + half slopeh = __float2half(1.0f); + + // ALiBi + if (max_bias > 0.0f) { + const int h = blockIdx.y; + + const float base = h < n_head_log2 ? m0 : m1; + const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slopeh = __float2half(powf(base, exph)); + } + + static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); + constexpr int nwarps = D / WARP_SIZE; + const int tid = WARP_SIZE*threadIdx.y + threadIdx.x; + __builtin_assume(tid < D); + + __shared__ half KQ[ncols*D]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + KQ[j*D + tid] = -HALF_MAX_HALF; + } + half2 * KQ2 = (half2 *) KQ; + + half kqmax[ncols]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + kqmax[j] = -HALF_MAX_HALF; + } + half kqsum[ncols] = {0.0f}; + + __shared__ half kqmax_shared[ncols][WARP_SIZE]; + __shared__ half kqsum_shared[ncols][WARP_SIZE]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + if (threadIdx.y == 0) { + kqmax_shared[j][threadIdx.x] = -HALF_MAX_HALF; + kqsum_shared[j][threadIdx.x] = 0.0f; + } + } + __syncthreads(); + + // Convert Q to half2 and store in registers: + half2 Q_h2[ncols][D/(2*WARP_SIZE)]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + const float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i]; + Q_h2[j][i0/WARP_SIZE] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y); + } + } + + half2 VKQ[ncols] = {{0.0f, 0.0f}}; + + const int k_start = parallel_blocks == 1 ? 0 : ip*D; + for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) { + // Calculate KQ tile and keep track of new maximum KQ values: + + // For unknown reasons using a half array of size 1 for kqmax_new causes a performance regression, + // see https://github.com/ggerganov/llama.cpp/pull/7061 . + // Therefore this variable is defined twice but only used once (so that the compiler can optimize out the unused variable). + half kqmax_new = kqmax[0]; + half kqmax_new_arr[ncols]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + kqmax_new_arr[j] = kqmax[j]; + } + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) { + const int i_KQ = i_KQ_0 + threadIdx.y; + + if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) { + break; + } + + half2 sum2[ncols] = {{0.0f, 0.0f}}; +#pragma unroll + for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) { + const int k_KQ = k_KQ_0 + threadIdx.x; + + const half2 K_ik = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + sum2[j] += K_ik * Q_h2[j][k_KQ_0/WARP_SIZE]; + } + } + +#pragma unroll + for (int j = 0; j < ncols; ++j) { + sum2[j] = warp_reduce_sum(sum2[j]); + half sum = __low2half(sum2[j]) + __high2half(sum2[j]); + sum += mask ? slopeh*maskh[j*ne11 + k_VKQ_0 + i_KQ] : __float2half(0.0f); + + if (ncols == 1) { + kqmax_new = ggml_cuda_hmax(kqmax_new, sum); + } else { + kqmax_new_arr[j] = ggml_cuda_hmax(kqmax_new_arr[j], sum); + } + + if (threadIdx.x == 0) { + KQ[j*D + i_KQ] = sum; + } + } + } + +#pragma unroll + for (int j = 0; j < ncols; ++j) { + half kqmax_new_j = ncols == 1 ? kqmax_new : kqmax_new_arr[j]; + + kqmax_new_j = warp_reduce_max(kqmax_new_j); + if (threadIdx.x == 0) { + kqmax_shared[j][threadIdx.y] = kqmax_new_j; + } + } + + __syncthreads(); + +#pragma unroll + for (int j = 0; j < ncols; ++j) { + half kqmax_new_j = kqmax_shared[j][threadIdx.x]; + kqmax_new_j = warp_reduce_max(kqmax_new_j); + + const half KQ_max_scale = hexp(kqmax[j] - kqmax_new_j); + kqmax[j] = kqmax_new_j; + + const half val = hexp(KQ[j*D + tid] - kqmax[j]); + kqsum[j] = kqsum[j]*KQ_max_scale + val; + KQ[j*D + tid] = val; + + VKQ[j] *= __half2half2(KQ_max_scale); + } + + __syncthreads(); + +#pragma unroll + for (int k0 = 0; k0 < D; k0 += 2) { + if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k0 >= ne11) { + break; + } + + half2 V_k; + reinterpret_cast(V_k.x) = V_h[(k_VKQ_0 + k0 + 0)*stride_KV + tid]; + reinterpret_cast(V_k.y) = V_h[(k_VKQ_0 + k0 + 1)*stride_KV + tid]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + VKQ[j] += V_k*KQ2[j*(D/2) + k0/2]; + } + } + + __syncthreads(); + } + +#pragma unroll + for (int j = 0; j < ncols; ++j) { + kqsum[j] = warp_reduce_sum(kqsum[j]); + if (threadIdx.x == 0) { + kqsum_shared[j][threadIdx.y] = kqsum[j]; + } + } + + __syncthreads(); + +#pragma unroll + for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) { + kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x]; + kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]); + + half dst_val = (__low2half(VKQ[j_VKQ]) + __high2half(VKQ[j_VKQ])); + if (parallel_blocks == 1) { + dst_val /= kqsum[j_VKQ]; + } + const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; + dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; + } + + if (parallel_blocks != 1 && tid != 0) { +#pragma unroll + for (int j = 0; j < ncols; ++j) { + dst_meta[(ic0 + j)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j], kqsum[j]); + } + } +#else + NO_DEVICE_CODE; +#endif // FP16_AVAILABLE +} + +template void launch_fattn_vec_f16( + const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, + ggml_cuda_pool & pool, cudaStream_t main_stream +) { + ggml_cuda_pool_alloc dst_tmp(pool); + ggml_cuda_pool_alloc dst_tmp_meta(pool); + + if (parallel_blocks > 1) { + dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); + dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); + } + + constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE; + const dim3 block_dim(WARP_SIZE, nwarps, 1); + const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); + const int shmem = 0; + + float scale = 1.0f; + float max_bias = 0.0f; + + memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); + memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); + + const uint32_t n_head = Q->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + flash_attn_vec_ext_f16 + <<>> ( + (const char *) Q->data, + (const char *) K->data, + (const char *) V->data, + mask ? ((const char *) mask->data) : nullptr, + parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, + scale, max_bias, m0, m1, n_head_log2, + Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], + K->ne[0], K->ne[1], K->ne[2], K->ne[3], + mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, + Q->nb[1], Q->nb[2], Q->nb[3], + K->nb[1], K->nb[2], K->nb[3], + KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] + ); + CUDA_CHECK(cudaGetLastError()); + + if (parallel_blocks == 1) { + return; + } + + const dim3 block_dim_combine(D, 1, 1); + const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); + const int shmem_combine = 0; + + flash_attn_combine_results + <<>> + (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); + CUDA_CHECK(cudaGetLastError()); +} + +void ggml_cuda_flash_attn_ext_vec_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + const ggml_tensor * mask = dst->src[3]; + + ggml_tensor * KQV = dst; + + const int32_t precision = KQV->op_params[2]; + GGML_ASSERT(precision == GGML_PREC_DEFAULT); + + constexpr int cols_per_block = 1; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 256: + launch_fattn_vec_f16<256, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } +} + +void ggml_cuda_flash_attn_ext_vec_f16_no_mma(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + const ggml_tensor * mask = dst->src[3]; + + ggml_tensor * KQV = dst; + + const int32_t precision = KQV->op_params[2]; + GGML_ASSERT(precision == GGML_PREC_DEFAULT); + GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); + + if (Q->ne[1] == 1) { + constexpr int cols_per_block = 1; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + if (Q->ne[1] == 2) { + constexpr int cols_per_block = 2; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + if (Q->ne[1] <= 4) { + constexpr int cols_per_block = 4; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + if (Q->ne[1] <= 8) { + constexpr int cols_per_block = 8; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + constexpr int cols_per_block = 8; + constexpr int parallel_blocks = 1; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } +} diff --git a/ggml-cuda/fattn-vec-f16.cuh b/ggml-cuda/fattn-vec-f16.cuh new file mode 100644 index 000000000..c7023610a --- /dev/null +++ b/ggml-cuda/fattn-vec-f16.cuh @@ -0,0 +1,5 @@ +#include "common.cuh" + +void ggml_cuda_flash_attn_ext_vec_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +void ggml_cuda_flash_attn_ext_vec_f16_no_mma(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml-cuda/fattn-vec-f32.cu b/ggml-cuda/fattn-vec-f32.cu new file mode 100644 index 000000000..40c336ce3 --- /dev/null +++ b/ggml-cuda/fattn-vec-f32.cu @@ -0,0 +1,384 @@ +#include "common.cuh" +#include "fattn-common.cuh" +#include "fattn-vec-f32.cuh" + +template // D == head size +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +__launch_bounds__(D, 1) +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +static __global__ void flash_attn_vec_ext_f32( + const char * __restrict__ Q, + const char * __restrict__ K, + const char * __restrict__ V, + const char * __restrict__ mask, + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, + const float max_bias, + const float m0, + const float m1, + const uint32_t n_head_log2, + const int ne00, + const int ne01, + const int ne02, + const int ne03, + const int ne10, + const int ne11, + const int ne12, + const int ne13, + const int ne31, + const int nb31, + const int nb01, + const int nb02, + const int nb03, + const int nb11, + const int nb12, + const int nb13, + const int ne0, + const int ne1, + const int ne2, + const int ne3) { + //In this kernel Q, K, V are matrices while i, j, k are matrix indices. + + const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. + const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + + const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. + const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); + const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); + const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const half * maskh = (const half *) mask + ne11*ic0; + + const int stride_KV = nb11 / sizeof(half); + const int stride_KV2 = nb11 / sizeof(half2); + + float slope = 1.0f; + + // ALiBi + if (max_bias > 0.0f) { + const int h = blockIdx.y; + + const float base = h < n_head_log2 ? m0 : m1; + const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slope = powf(base, exph); + } + + static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); + constexpr int nwarps = D / WARP_SIZE; + const int tid = WARP_SIZE*threadIdx.y + threadIdx.x; + __builtin_assume(tid < D); + + __shared__ float KQ[ncols*D]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + KQ[j*D + tid] = -FLT_MAX/2.0f; + } + + float kqmax[ncols]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + kqmax[j] = -FLT_MAX/2.0f; + } + float kqsum[ncols] = {0.0f}; + + __shared__ float kqmax_shared[ncols][WARP_SIZE]; + __shared__ float kqsum_shared[ncols][WARP_SIZE]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + if (threadIdx.y == 0) { + kqmax_shared[j][threadIdx.x] = -FLT_MAX/2.0f; + kqsum_shared[j][threadIdx.x] = 0.0f; + } + } + __syncthreads(); + + // Convert Q to half2 and store in registers: + float2 Q_h2[ncols][D/(2*WARP_SIZE)]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + Q_h2[j][i0/WARP_SIZE] = Q_f2[j*(nb01/sizeof(float2)) + i]; + Q_h2[j][i0/WARP_SIZE].x *= scale; + Q_h2[j][i0/WARP_SIZE].y *= scale; + } + } + + float VKQ[ncols] = {0.0f}; + + const int k_start = parallel_blocks == 1 ? 0 : ip*D; + for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) { + // Calculate KQ tile and keep track of new maximum KQ values: + + float kqmax_new_arr[ncols]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + kqmax_new_arr[j] = kqmax[j]; + } + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) { + const int i_KQ = i_KQ_0 + threadIdx.y; + + if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) { + break; + } + + float sum[ncols] = {0.0f}; +#pragma unroll + for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) { + const int k_KQ = k_KQ_0 + threadIdx.x; + + const half2 K_ik = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ]; +#pragma unroll + for (int j = 0; j < ncols; ++j) { + sum[j] += __low2float(K_ik) * Q_h2[j][k_KQ_0/WARP_SIZE].x; + sum[j] += __high2float(K_ik) * Q_h2[j][k_KQ_0/WARP_SIZE].y; + } + } + +#pragma unroll + for (int j = 0; j < ncols; ++j) { + sum[j] = warp_reduce_sum(sum[j]); + sum[j] += mask ? slope*__half2float(maskh[j*ne11 + k_VKQ_0 + i_KQ]) : 0.0f; + + kqmax_new_arr[j] = fmaxf(kqmax_new_arr[j], sum[j]); + + if (threadIdx.x == 0) { + KQ[j*D + i_KQ] = sum[j]; + } + } + } + +#pragma unroll + for (int j = 0; j < ncols; ++j) { + float kqmax_new_j = kqmax_new_arr[j]; + + kqmax_new_j = warp_reduce_max(kqmax_new_j); + if (threadIdx.x == 0) { + kqmax_shared[j][threadIdx.y] = kqmax_new_j; + } + } + + __syncthreads(); + +#pragma unroll + for (int j = 0; j < ncols; ++j) { + float kqmax_new_j = kqmax_shared[j][threadIdx.x]; + kqmax_new_j = warp_reduce_max(kqmax_new_j); + + const float KQ_max_scale = expf(kqmax[j] - kqmax_new_j); + kqmax[j] = kqmax_new_j; + + const float val = expf(KQ[j*D + tid] - kqmax[j]); + kqsum[j] = kqsum[j]*KQ_max_scale + val; + KQ[j*D + tid] = val; + + VKQ[j] *= KQ_max_scale; + } + + __syncthreads(); + +#pragma unroll + for (int k = 0; k < D; ++k) { + if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k >= ne11) { + break; + } + + const float V_ki = __half2float(V_h[(k_VKQ_0 + k)*stride_KV + tid]); +#pragma unroll + for (int j = 0; j < ncols; ++j) { + VKQ[j] += V_ki*KQ[j*D + k]; + } + } + + __syncthreads(); + } + +#pragma unroll + for (int j = 0; j < ncols; ++j) { + kqsum[j] = warp_reduce_sum(kqsum[j]); + if (threadIdx.x == 0) { + kqsum_shared[j][threadIdx.y] = kqsum[j]; + } + } + + __syncthreads(); + +#pragma unroll + for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) { + kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x]; + kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]); + + float dst_val = VKQ[j_VKQ]; + if (parallel_blocks == 1) { + dst_val /= kqsum[j_VKQ]; + } + const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; + dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; + } + + if (parallel_blocks != 1 && tid != 0) { +#pragma unroll + for (int j = 0; j < ncols; ++j) { + dst_meta[(ic0 + j)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j], kqsum[j]); + } + } +} + +template void launch_fattn_vec_f32( + const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, + ggml_cuda_pool & pool, cudaStream_t main_stream +) { + ggml_cuda_pool_alloc dst_tmp(pool); + ggml_cuda_pool_alloc dst_tmp_meta(pool); + + if (parallel_blocks > 1) { + dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); + dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); + } + + constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE; + const dim3 block_dim(WARP_SIZE, nwarps, 1); + const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); + const int shmem = 0; + + float scale = 1.0f; + float max_bias = 0.0f; + + memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); + memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); + + const uint32_t n_head = Q->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + flash_attn_vec_ext_f32 + <<>> ( + (const char *) Q->data, + (const char *) K->data, + (const char *) V->data, + mask ? ((const char *) mask->data) : nullptr, + parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, + scale, max_bias, m0, m1, n_head_log2, + Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], + K->ne[0], K->ne[1], K->ne[2], K->ne[3], + mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, + Q->nb[1], Q->nb[2], Q->nb[3], + K->nb[1], K->nb[2], K->nb[3], + KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] + ); + CUDA_CHECK(cudaGetLastError()); + + if (parallel_blocks == 1) { + return; + } + + const dim3 block_dim_combine(D, 1, 1); + const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); + const int shmem_combine = 0; + + flash_attn_combine_results + <<>> + (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); + CUDA_CHECK(cudaGetLastError()); +} + +void ggml_cuda_flash_attn_ext_vec_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + const ggml_tensor * mask = dst->src[3]; + + ggml_tensor * KQV = dst; + + GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); + + if (Q->ne[1] == 1) { + constexpr int cols_per_block = 1; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + if (Q->ne[1] == 2) { + constexpr int cols_per_block = 2; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + if (Q->ne[1] <= 4) { + constexpr int cols_per_block = 4; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + if (Q->ne[1] <= 8) { + constexpr int cols_per_block = 8; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + constexpr int cols_per_block = 8; + constexpr int parallel_blocks = 1; + switch (Q->ne[0]) { + case 64: + launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } +} diff --git a/ggml-cuda/fattn-vec-f32.cuh b/ggml-cuda/fattn-vec-f32.cuh new file mode 100644 index 000000000..614d54ae3 --- /dev/null +++ b/ggml-cuda/fattn-vec-f32.cuh @@ -0,0 +1,3 @@ +#include "common.cuh" + +void ggml_cuda_flash_attn_ext_vec_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml-cuda/fattn.cu b/ggml-cuda/fattn.cu index ac5d6672b..419f8e752 100644 --- a/ggml-cuda/fattn.cu +++ b/ggml-cuda/fattn.cu @@ -1,4 +1,7 @@ #include "common.cuh" +#include "fattn-common.cuh" +#include "fattn-vec-f16.cuh" +#include "fattn-vec-f32.cuh" #include "fattn.cuh" #include @@ -7,251 +10,6 @@ #include #endif -#define FATTN_KQ_STRIDE 256 -#define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction. -#define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs. - -template // D == head size -#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) -__launch_bounds__(D, 1) -#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) -static __global__ void flash_attn_vec_ext_f16( - const char * __restrict__ Q, - const char * __restrict__ K, - const char * __restrict__ V, - const char * __restrict__ mask, - float * __restrict__ dst, - float2 * __restrict__ dst_meta, - const float scale, - const float max_bias, - const float m0, - const float m1, - const uint32_t n_head_log2, - const int ne00, - const int ne01, - const int ne02, - const int ne03, - const int ne10, - const int ne11, - const int ne12, - const int ne13, - const int ne31, - const int nb31, - const int nb01, - const int nb02, - const int nb03, - const int nb11, - const int nb12, - const int nb13, - const int ne0, - const int ne1, - const int ne2, - const int ne3) { -#if FP16_AVAILABLE - //In this kernel Q, K, V are matrices while i, j, k are matrix indices. - - const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. - const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. - - const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. - const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); - const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); - const half * V_h = (const half *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape - const half * maskh = (const half *) mask + ne11*ic0; - - const int stride_KV = nb11 / sizeof(half); - const int stride_KV2 = nb11 / sizeof(half2); - - half slopeh = __float2half(1.0f); - - // ALiBi - if (max_bias > 0.0f) { - const int h = blockIdx.y; - - const float base = h < n_head_log2 ? m0 : m1; - const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; - - slopeh = __float2half(powf(base, exph)); - } - - static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); - constexpr int nwarps = D / WARP_SIZE; - const int tid = WARP_SIZE*threadIdx.y + threadIdx.x; - __builtin_assume(tid < D); - - __shared__ half KQ[ncols*D]; -#pragma unroll - for (int j = 0; j < ncols; ++j) { - KQ[j*D + tid] = -HALF_MAX_HALF; - } - half2 * KQ2 = (half2 *) KQ; - - half kqmax[ncols]; -#pragma unroll - for (int j = 0; j < ncols; ++j) { - kqmax[j] = -HALF_MAX_HALF; - } - half kqsum[ncols] = {0.0f}; - - __shared__ half kqmax_shared[ncols][WARP_SIZE]; - __shared__ half kqsum_shared[ncols][WARP_SIZE]; -#pragma unroll - for (int j = 0; j < ncols; ++j) { - if (threadIdx.y == 0) { - kqmax_shared[j][threadIdx.x] = -HALF_MAX_HALF; - kqsum_shared[j][threadIdx.x] = 0.0f; - } - } - __syncthreads(); - - // Convert Q to half2 and store in registers: - half2 Q_h2[ncols][D/(2*WARP_SIZE)]; -#pragma unroll - for (int j = 0; j < ncols; ++j) { -#pragma unroll - for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { - const int i = i0 + threadIdx.x; - - const float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i]; - Q_h2[j][i0/WARP_SIZE] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y); - } - } - - half2 VKQ[ncols] = {{0.0f, 0.0f}}; - - const int k_start = parallel_blocks == 1 ? 0 : ip*D; - for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*D) { - // Calculate KQ tile and keep track of new maximum KQ values: - - // For unknown reasons using a half array of size 1 for kqmax_new causes a performance regression, - // see https://github.com/ggerganov/llama.cpp/pull/7061 . - // Therefore this variable is defined twice but only used once (so that the compiler can optimize out the unused variable). - half kqmax_new = kqmax[0]; - half kqmax_new_arr[ncols]; -#pragma unroll - for (int j = 0; j < ncols; ++j) { - kqmax_new_arr[j] = kqmax[j]; - } - -#pragma unroll - for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += nwarps) { - const int i_KQ = i_KQ_0 + threadIdx.y; - - if ((i_KQ_0 + nwarps > D && i_KQ >= D) || (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + i_KQ >= ne11)) { - break; - } - - half2 sum2[ncols] = {{0.0f, 0.0f}}; -#pragma unroll - for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) { - const int k_KQ = k_KQ_0 + threadIdx.x; - - const half2 K_ik = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ]; -#pragma unroll - for (int j = 0; j < ncols; ++j) { - sum2[j] += K_ik * Q_h2[j][k_KQ_0/WARP_SIZE]; - } - } - -#pragma unroll - for (int j = 0; j < ncols; ++j) { - sum2[j] = warp_reduce_sum(sum2[j]); - half sum = __low2half(sum2[j]) + __high2half(sum2[j]); - sum += mask ? slopeh*maskh[j*ne11 + k_VKQ_0 + i_KQ] : __float2half(0.0f); - - if (ncols == 1) { - kqmax_new = ggml_cuda_hmax(kqmax_new, sum); - } else { - kqmax_new_arr[j] = ggml_cuda_hmax(kqmax_new_arr[j], sum); - } - - if (threadIdx.x == 0) { - KQ[j*D + i_KQ] = sum; - } - } - } - -#pragma unroll - for (int j = 0; j < ncols; ++j) { - half kqmax_new_j = ncols == 1 ? kqmax_new : kqmax_new_arr[j]; - - kqmax_new_j = warp_reduce_max(kqmax_new_j); - if (threadIdx.x == 0) { - kqmax_shared[j][threadIdx.y] = kqmax_new_j; - } - } - - __syncthreads(); - -#pragma unroll - for (int j = 0; j < ncols; ++j) { - half kqmax_new_j = kqmax_shared[j][threadIdx.x]; - kqmax_new_j = warp_reduce_max(kqmax_new_j); - - const half KQ_max_scale = hexp(kqmax[j] - kqmax_new_j); - kqmax[j] = kqmax_new_j; - - const half val = hexp(KQ[j*D + tid] - kqmax[j]); - kqsum[j] = kqsum[j]*KQ_max_scale + val; - KQ[j*D + tid] = val; - - VKQ[j] *= __half2half2(KQ_max_scale); - } - - __syncthreads(); - -#pragma unroll - for (int k0 = 0; k0 < D; k0 += 2) { - if (FATTN_KQ_STRIDE % D != 0 && k_VKQ_0 + k0 >= ne11) { - break; - } - - half2 V_k; - reinterpret_cast(V_k.x) = V_h[(k_VKQ_0 + k0 + 0)*stride_KV + tid]; - reinterpret_cast(V_k.y) = V_h[(k_VKQ_0 + k0 + 1)*stride_KV + tid]; -#pragma unroll - for (int j = 0; j < ncols; ++j) { - VKQ[j] += V_k*KQ2[j*(D/2) + k0/2]; - } - } - - __syncthreads(); - } - -#pragma unroll - for (int j = 0; j < ncols; ++j) { - kqsum[j] = warp_reduce_sum(kqsum[j]); - if (threadIdx.x == 0) { - kqsum_shared[j][threadIdx.y] = kqsum[j]; - } - } - - __syncthreads(); - -#pragma unroll - for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) { - kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x]; - kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]); - - half dst_val = (__low2half(VKQ[j_VKQ]) + __high2half(VKQ[j_VKQ])); - if (parallel_blocks == 1) { - dst_val /= kqsum[j_VKQ]; - } - const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; - dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; - } - - if (parallel_blocks != 1 && tid != 0) { -#pragma unroll - for (int j = 0; j < ncols; ++j) { - dst_meta[(ic0 + j)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j], kqsum[j]); - } - } -#else - NO_DEVICE_CODE; -#endif // FP16_AVAILABLE -} - // D == head size, VKQ_stride == num VKQ rows calculated in parallel: template #if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) @@ -655,54 +413,6 @@ static __global__ void flash_attn_ext_f16( #endif // FP16_MMA_AVAILABLE } -template // D == head size -#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) -__launch_bounds__(D, 1) -#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) -static __global__ void flash_attn_combine_results( - const float * __restrict__ VKQ_parts, - const float2 * __restrict__ VKQ_meta, - float * __restrict__ dst) { -#if FP16_AVAILABLE - VKQ_parts += parallel_blocks*D * gridDim.y*blockIdx.x; - VKQ_meta += parallel_blocks * gridDim.y*blockIdx.x; - dst += D * gridDim.y*blockIdx.x; - - const int tid = threadIdx.x; - __builtin_assume(tid < D); - - __shared__ float2 meta[parallel_blocks]; - if (tid < 2*parallel_blocks) { - ((float *) meta)[threadIdx.x] = ((const float *)VKQ_meta) [blockIdx.y*(2*parallel_blocks) + tid]; - } - - __syncthreads(); - - float kqmax = meta[0].x; -#pragma unroll - for (int l = 1; l < parallel_blocks; ++l) { - kqmax = max(kqmax, meta[l].x); - } - - float VKQ_numerator = 0.0f; - float VKQ_denominator = 0.0f; -#pragma unroll - for (int l = 0; l < parallel_blocks; ++l) { - const float diff = meta[l].x - kqmax; - const float KQ_max_scale = expf(diff); - const uint32_t ftz_mask = 0xFFFFFFFF * (diff > SOFTMAX_FTZ_THRESHOLD); - *((uint32_t *) &KQ_max_scale) &= ftz_mask; - - VKQ_numerator += KQ_max_scale * VKQ_parts[l*gridDim.y*D + blockIdx.y*D + tid]; - VKQ_denominator += KQ_max_scale * meta[l].y; - } - - dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator; -#else - NO_DEVICE_CODE; -#endif // FP16_AVAILABLE -} - constexpr int get_max_power_of_2(int x) { return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1; } @@ -727,66 +437,6 @@ static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed."); static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed."); static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed."); -template void launch_fattn_vec_f16( - const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, - ggml_cuda_pool & pool, cudaStream_t main_stream -) { - ggml_cuda_pool_alloc dst_tmp(pool); - ggml_cuda_pool_alloc dst_tmp_meta(pool); - - if (parallel_blocks > 1) { - dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); - dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); - } - - constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE; - const dim3 block_dim(WARP_SIZE, nwarps, 1); - const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); - const int shmem = 0; - - float scale = 1.0f; - float max_bias = 0.0f; - - memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); - - const uint32_t n_head = Q->ne[2]; - const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); - - const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - - flash_attn_vec_ext_f16 - <<>> ( - (const char *) Q->data, - (const char *) K->data, - (const char *) V->data, - mask ? ((const char *) mask->data) : nullptr, - parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, - scale, max_bias, m0, m1, n_head_log2, - Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], - K->ne[0], K->ne[1], K->ne[2], K->ne[3], - mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, - Q->nb[1], Q->nb[2], Q->nb[3], - K->nb[1], K->nb[2], K->nb[3], - KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] - ); - CUDA_CHECK(cudaGetLastError()); - - if (parallel_blocks == 1) { - return; - } - - const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); - const int shmem_combine = 0; - - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); - CUDA_CHECK(cudaGetLastError()); -} - template void launch_fattn_f16_impl( const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, ggml_cuda_pool & pool, cudaStream_t main_stream @@ -891,95 +541,22 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst const int32_t precision = KQV->op_params[2]; + if (!fast_fp16_available(cc)) { + ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); + return; + } + if (!fp16_mma_available(cc)) { - GGML_ASSERT(precision == GGML_PREC_DEFAULT); - GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); - - if (Q->ne[1] == 1) { - constexpr int cols_per_block = 1; - constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } - return; - } - - if (Q->ne[1] == 2) { - constexpr int cols_per_block = 2; - constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } - return; - } - - if (Q->ne[1] <= 4) { - constexpr int cols_per_block = 4; - constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } - return; - } - - if (Q->ne[1] <= 8) { - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } - return; - } - - constexpr int cols_per_block = 8; - constexpr int parallel_blocks = 1; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + ggml_cuda_flash_attn_ext_vec_f16_no_mma(ctx, dst); return; } if (precision != GGML_PREC_DEFAULT) { + if (Q->ne[1] == 1 && (Q->ne[0] == 64 || Q->ne[0] == 128)) { + ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); + return; + } + if (Q->ne[1] <= 32 || Q->ne[0] > 128) { constexpr int cols_per_block = 16; constexpr int nwarps = 4; @@ -1037,22 +614,7 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst } if (Q->ne[1] == 1 && Q->ne[0] % (2*WARP_SIZE) == 0) { - constexpr int cols_per_block = 1; - constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 256: - launch_fattn_vec_f16<256, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); return; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 731788b95..45a2cb85a 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2,6 +2,7 @@ #include #include #include + #include #include #include @@ -2173,11 +2174,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_timestep_embedding()); test_cases.emplace_back(new test_leaky_relu()); -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) - for (int hs : { 64, 128, }) { // other head sizes not implemented -#else for (int hs : { 64, 80, 128, 256, }) { -#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) for (float max_bias : {0.0f, 8.0f}) { for (int nh : { 32, }) { for (int kv : { 512, 1024, }) { From 0d5cef78aeafae4d4e6d56e2d4bcda771af58cc9 Mon Sep 17 00:00:00 2001 From: Neo Zhang <14088817+arthw@users.noreply.github.com> Date: Mon, 13 May 2024 08:02:55 +0800 Subject: [PATCH 002/137] [SYCL] update CI with oneapi 2024.1 (#7235) Co-authored-by: Zhang --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2d747e688..3408b74ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -898,7 +898,7 @@ jobs: shell: bash env: - WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/62641e01-1e8d-4ace-91d6-ae03f7f8a71f/w_BaseKit_p_2024.0.0.49563_offline.exe + WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/7dff44ba-e3af-4448-841c-0d616c8da6e7/w_BaseKit_p_2024.1.0.595_offline.exe WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel steps: From cbf75894d256f1861f6409565db599365de3d4b8 Mon Sep 17 00:00:00 2001 From: Neo Zhang <14088817+arthw@users.noreply.github.com> Date: Mon, 13 May 2024 08:04:29 +0800 Subject: [PATCH 003/137] [SYCL] Add oneapi runtime dll files to win release package (#7241) * add oneapi running time dlls to release package * fix path * fix path * fix path * fix path * fix path --------- Co-authored-by: Zhang --- .github/workflows/build.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3408b74ac..8c338534d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -900,7 +900,7 @@ jobs: env: WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/7dff44ba-e3af-4448-841c-0d616c8da6e7/w_BaseKit_p_2024.1.0.595_offline.exe WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel - + ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI" steps: - name: Clone id: checkout @@ -932,6 +932,17 @@ jobs: id: pack_artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} run: | + echo "cp oneAPI running time dll files in ${{ env.ONEAPI_ROOT }} to ./build/bin" + cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_sycl_blas.4.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_core.2.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/mkl/latest/bin/mkl_tbb_thread.2.dll" ./build/bin + + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/pi_win_proxy_loader.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/pi_level_zero.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/sycl7.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/svml_dispmd.dll" ./build/bin + cp "${{ env.ONEAPI_ROOT }}/compiler/latest/bin/libmmd.dll" ./build/bin + echo "cp oneAPI running time dll files to ./build/bin done" 7z a llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip ./build/bin/* - name: Upload artifacts From e586ee42595500c53938e937b6b6ad5353ad76dc Mon Sep 17 00:00:00 2001 From: Benjamin Findley <39356821+Kartoffelsaft@users.noreply.github.com> Date: Sun, 12 May 2024 19:40:08 -0700 Subject: [PATCH 004/137] change default temperature of OAI compat API from 0 to 1 (#7226) * change default temperature of OAI compat API from 0 to 1 * make tests explicitly send temperature to OAI API --- examples/server/tests/features/steps/steps.py | 7 +++++-- examples/server/utils.hpp | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index f4b1ac1d7..577b87af3 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -887,6 +887,7 @@ async def oai_chat_completions(user_prompt, base_path, async_client, debug=False, + temperature=None, model=None, n_predict=None, enable_streaming=None, @@ -913,7 +914,8 @@ async def oai_chat_completions(user_prompt, "model": model, "max_tokens": n_predict, "stream": enable_streaming, - "seed": seed + "temperature": temperature if temperature is not None else 0.0, + "seed": seed, } if response_format is not None: payload['response_format'] = response_format @@ -978,7 +980,8 @@ async def oai_chat_completions(user_prompt, max_tokens=n_predict, stream=enable_streaming, response_format=payload.get('response_format'), - seed=seed + seed=seed, + temperature=payload['temperature'] ) except openai.error.AuthenticationError as e: if expect_api_error is not None and expect_api_error: diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index d872b63f5..d8a2286e4 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -371,7 +371,7 @@ static json oaicompat_completion_params_parse( llama_params["presence_penalty"] = json_value(body, "presence_penalty", 0.0); llama_params["seed"] = json_value(body, "seed", LLAMA_DEFAULT_SEED); llama_params["stream"] = json_value(body, "stream", false); - llama_params["temperature"] = json_value(body, "temperature", 0.0); + llama_params["temperature"] = json_value(body, "temperature", 1.0); llama_params["top_p"] = json_value(body, "top_p", 1.0); // Apply chat template to the list of messages From b1f8af1886e8187db6bb2a9b87cfc1c0f175f629 Mon Sep 17 00:00:00 2001 From: Brian Date: Mon, 13 May 2024 12:56:47 +1000 Subject: [PATCH 005/137] convert.py: Outfile default name change and additional metadata support (#4858) * convert.py: Outfile default name change and additional metadata support * convert.py: don't stringify Metadata load method output * convert.py: typo fix * convert.py: fix metadata format to sync with LLM_KV_NAMES in llama.cpp --- convert.py | 178 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 154 insertions(+), 24 deletions(-) diff --git a/convert.py b/convert.py index 148bfd66a..e2e642351 100755 --- a/convert.py +++ b/convert.py @@ -24,7 +24,7 @@ from abc import ABC, abstractmethod from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, ClassVar, IO, Iterable, Literal, Protocol, TypeVar, runtime_checkable +from typing import TYPE_CHECKING, Any, Callable, ClassVar, IO, Iterable, Literal, Protocol, TypeVar, runtime_checkable, Optional import numpy as np from sentencepiece import SentencePieceProcessor @@ -344,10 +344,47 @@ class Params: return params +@dataclass +class Metadata: + name: Optional[str] = None + author: Optional[str] = None + version: Optional[str] = None + url: Optional[str] = None + description: Optional[str] = None + licence: Optional[str] = None + source_url: Optional[str] = None + source_hf_repo: Optional[str] = None + + @staticmethod + def load(metadata_path: Path) -> Metadata: + if metadata_path is None or not metadata_path.exists(): + return Metadata() + + with open(metadata_path, 'r') as file: + data = json.load(file) + + # Create a new Metadata instance + metadata = Metadata() + + # Assigning values to Metadata attributes if they exist in the JSON file + # This is based on LLM_KV_NAMES mapping in llama.cpp + metadata.name = data.get("general.name") + metadata.author = data.get("general.author") + metadata.version = data.get("general.version") + metadata.url = data.get("general.url") + metadata.description = data.get("general.description") + metadata.license = data.get("general.license") + metadata.source_url = data.get("general.source.url") + metadata.source_hf_repo = data.get("general.source.huggingface.repository") + + return metadata + + # # vocab # + @runtime_checkable class BaseVocab(Protocol): tokenizer_model: ClassVar[str] @@ -1066,21 +1103,42 @@ class OutputFile: def __init__(self, fname_out: Path, endianess:gguf.GGUFEndian = gguf.GGUFEndian.LITTLE): self.gguf = gguf.GGUFWriter(fname_out, gguf.MODEL_ARCH_NAMES[ARCH], endianess=endianess) - def add_meta_arch(self, params: Params) -> None: + def add_meta_model(self, params: Params, metadata: Metadata) -> None: + # Metadata About The Model And Its Provenence name = "LLaMA" - - # TODO: better logic to determine model name - if params.n_ctx == 4096: - name = "LLaMA v2" + if metadata is not None and metadata.name is not None: + name = metadata.name elif params.path_model is not None: - name = str(params.path_model.parent).split('/')[-1] + name = str(params.path_model.parent).split("/")[-1] + elif params.n_ctx == 4096: + # Heuristic detection of LLaMA v2 model + name = "LLaMA v2" - self.gguf.add_name (name) - self.gguf.add_vocab_size (params.n_vocab) - self.gguf.add_context_length (params.n_ctx) - self.gguf.add_embedding_length (params.n_embd) - self.gguf.add_block_count (params.n_layer) - self.gguf.add_feed_forward_length (params.n_ff) + self.gguf.add_name(name) + + if metadata is not None: + if metadata.author is not None: + self.gguf.add_author(metadata.author) + if metadata.version is not None: + self.gguf.add_version(metadata.version) + if metadata.url is not None: + self.gguf.add_url(metadata.url) + if metadata.description is not None: + self.gguf.add_description(metadata.description) + if metadata.licence is not None: + self.gguf.add_licence(metadata.licence) + if metadata.source_url is not None: + self.gguf.add_source_url(metadata.source_url) + if metadata.source_hf_repo is not None: + self.gguf.add_source_hf_repo(metadata.source_hf_repo) + + def add_meta_arch(self, params: Params) -> None: + # Metadata About The Neural Architecture Itself + self.gguf.add_vocab_size(params.n_vocab) + self.gguf.add_context_length(params.n_ctx) + self.gguf.add_embedding_length(params.n_embd) + self.gguf.add_block_count(params.n_layer) + self.gguf.add_feed_forward_length(params.n_ff) self.gguf.add_rope_dimension_count(params.n_embd // params.n_head) self.gguf.add_head_count (params.n_head) self.gguf.add_head_count_kv (params.n_head_kv) @@ -1183,13 +1241,14 @@ class OutputFile: @staticmethod def write_vocab_only( fname_out: Path, params: Params, vocab: Vocab, svocab: gguf.SpecialVocab, - endianess: gguf.GGUFEndian = gguf.GGUFEndian.LITTLE, pad_vocab: bool = False, + endianess: gguf.GGUFEndian = gguf.GGUFEndian.LITTLE, pad_vocab: bool = False, metadata: Metadata = None, ) -> None: check_vocab_size(params, vocab, pad_vocab=pad_vocab) of = OutputFile(fname_out, endianess=endianess) # meta data + of.add_meta_model(params, metadata) of.add_meta_arch(params) of.add_meta_vocab(vocab) of.add_meta_special_vocab(svocab) @@ -1216,12 +1275,14 @@ class OutputFile: fname_out: Path, ftype: GGMLFileType, params: Params, model: LazyModel, vocab: BaseVocab, svocab: gguf.SpecialVocab, concurrency: int = DEFAULT_CONCURRENCY, endianess: gguf.GGUFEndian = gguf.GGUFEndian.LITTLE, pad_vocab: bool = False, + metadata: Metadata = None, ) -> None: check_vocab_size(params, vocab, pad_vocab=pad_vocab) of = OutputFile(fname_out, endianess=endianess) # meta data + of.add_meta_model(params, metadata) of.add_meta_arch(params) if isinstance(vocab, Vocab): of.add_meta_vocab(vocab) @@ -1257,6 +1318,37 @@ def pick_output_type(model: LazyModel, output_type_str: str | None) -> GGMLFileT raise ValueError(f"Unexpected combination of types: {name_to_type}") +def model_parameter_count(model: LazyModel) -> int: + total_model_parameters = 0 + for i, (name, lazy_tensor) in enumerate(model.items()): + sum_weights_in_tensor = 1 + for dim in lazy_tensor.shape: + sum_weights_in_tensor *= dim + total_model_parameters += sum_weights_in_tensor + return total_model_parameters + + +def model_parameter_count_rounded_notation(model_params_count: int) -> str: + if model_params_count > 1e12 : + # Trillions Of Parameters + scaled_model_params = model_params_count * 1e-12 + scale_suffix = "T" + elif model_params_count > 1e9 : + # Billions Of Parameters + scaled_model_params = model_params_count * 1e-9 + scale_suffix = "B" + elif model_params_count > 1e6 : + # Millions Of Parameters + scaled_model_params = model_params_count * 1e-6 + scale_suffix = "M" + else: + # Thousands Of Parameters + scaled_model_params = model_params_count * 1e-3 + scale_suffix = "K" + + return f"{round(scaled_model_params)}{scale_suffix}" + + def convert_to_output_type(model: LazyModel, output_type: GGMLFileType) -> LazyModel: return {name: tensor.astype(output_type.type_for_tensor(name, tensor)) for (name, tensor) in model.items()} @@ -1436,13 +1528,35 @@ class VocabFactory: return vocab, special_vocab -def default_outfile(model_paths: list[Path], file_type: GGMLFileType) -> Path: - namestr = { - GGMLFileType.AllF32: "f32", - GGMLFileType.MostlyF16: "f16", - GGMLFileType.MostlyQ8_0:"q8_0", +def default_convention_outfile(file_type: GGMLFileType, params: Params, model_params_count: int, metadata: Metadata) -> str: + quantization = { + GGMLFileType.AllF32: "F32", + GGMLFileType.MostlyF16: "F16", + GGMLFileType.MostlyQ8_0: "Q8_0", }[file_type] - ret = model_paths[0].parent / f"ggml-model-{namestr}.gguf" + + parameters = model_parameter_count_rounded_notation(model_params_count) + + expert_count = "" + if params.n_experts is not None: + expert_count = f"{params.n_experts}x" + + version = "" + if metadata is not None and metadata.version is not None: + version = f"-{metadata.version}" + + name = "ggml-model" + if metadata is not None and metadata.name is not None: + name = metadata.name + elif params.path_model is not None: + name = params.path_model.name + + return f"{name}{version}-{expert_count}{parameters}-{quantization}" + + +def default_outfile(model_paths: list[Path], file_type: GGMLFileType, params: Params, model_params_count: int, metadata: Metadata) -> Path: + default_filename = default_convention_outfile(file_type, params, model_params_count, metadata) + ret = model_paths[0].parent / f"{default_filename}.gguf" if ret in model_paths: logger.error( f"Error: Default output path ({ret}) would overwrite the input. " @@ -1480,17 +1594,30 @@ def main(args_in: list[str] | None = None) -> None: parser.add_argument("--pad-vocab", action="store_true", help="add pad tokens when model vocab expects more than tokenizer metadata provides") parser.add_argument("--skip-unknown", action="store_true", help="skip unknown tensor names instead of failing") parser.add_argument("--verbose", action="store_true", help="increase output verbosity") + parser.add_argument("--metadata", type=Path, help="Specify the path for a metadata file") + parser.add_argument("--get-outfile", action="store_true", help="get calculated default outfile name") args = parser.parse_args(args_in) if args.verbose: logging.basicConfig(level=logging.DEBUG) - elif args.dump_single or args.dump: + elif args.dump_single or args.dump or args.get_outfile: # Avoid printing anything besides the dump output logging.basicConfig(level=logging.WARNING) else: logging.basicConfig(level=logging.INFO) + metadata = Metadata.load(args.metadata) + + if args.get_outfile: + model_plus = load_some_model(args.model) + params = Params.load(model_plus) + model = convert_model_names(model_plus.model, params, args.skip_unknown) + model_params_count = model_parameter_count(model_plus.model) + ftype = pick_output_type(model, args.outtype) + print(f"{default_convention_outfile(ftype, params, model_params_count, metadata)}") # noqa: NP100 + return + if args.no_vocab and args.vocab_only: raise ValueError("--vocab-only does not make sense with --no-vocab") @@ -1504,6 +1631,9 @@ def main(args_in: list[str] | None = None) -> None: else: model_plus = ModelPlus(model = {}, paths = [args.model / 'dummy'], format = 'none', vocab = None) + model_params_count = model_parameter_count(model_plus.model) + logger.info(f"model parameters count : {model_params_count} ({model_parameter_count_rounded_notation(model_params_count)})") + if args.dump: do_dump_model(model_plus) return @@ -1557,7 +1687,7 @@ def main(args_in: list[str] | None = None) -> None: f_norm_eps = 1e-5, ) OutputFile.write_vocab_only(outfile, params, vocab, special_vocab, - endianess=endianess, pad_vocab=args.pad_vocab) + endianess=endianess, pad_vocab=args.pad_vocab, metadata=metadata) logger.info(f"Wrote {outfile}") return @@ -1570,13 +1700,13 @@ def main(args_in: list[str] | None = None) -> None: model = convert_model_names(model, params, args.skip_unknown) ftype = pick_output_type(model, args.outtype) model = convert_to_output_type(model, ftype) - outfile = args.outfile or default_outfile(model_plus.paths, ftype) + outfile = args.outfile or default_outfile(model_plus.paths, ftype, params, model_params_count, metadata) params.ftype = ftype logger.info(f"Writing {outfile}, format {ftype}") OutputFile.write_all(outfile, ftype, params, model, vocab, special_vocab, - concurrency=args.concurrency, endianess=endianess, pad_vocab=args.pad_vocab) + concurrency=args.concurrency, endianess=endianess, pad_vocab=args.pad_vocab, metadata=metadata) logger.info(f"Wrote {outfile}") From 9aa672490c848e45eaa704a554e0f1f6df995fc8 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Date: Mon, 13 May 2024 10:35:14 +0200 Subject: [PATCH 006/137] llama : rename jina tokenizers to v2 (#7249) * refactor: rename jina tokenizers to v2 * refactor: keep refactoring non-breaking --- convert-hf-to-gguf-update.py | 6 +++--- convert-hf-to-gguf.py | 6 +++--- llama.cpp | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index cd2674a0e..14aa0c45a 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -74,9 +74,9 @@ models = [ {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", }, {"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", }, {"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", }, - {"name": "jina-en", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", }, # WPM! - {"name": "jina-es", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", }, - {"name": "jina-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, + {"name": "jina-v2-en", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", }, # WPM! + {"name": "jina-v2-es", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", }, + {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, ] # make directory "models/tokenizers" if it doesn't exist diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index ec7f4dd75..d6e5dece0 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -475,13 +475,13 @@ class Model: res = "dbrx" if chkhsh == "0876d13b50744004aa9aeae05e7b0647eac9d801b5ba4668afc01e709c15e19f": # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-en - res = "jina-en" + res = "jina-v2-en" if chkhsh == "171aeeedd6fb548d418a7461d053f11b6f1f1fc9b387bd66640d28a4b9f5c643": # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-es - res = "jina-es" + res = "jina-v2-es" if chkhsh == "27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6": # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-de - res = "jina-de" + res = "jina-v2-de" if res is None: logger.warning("\n") diff --git a/llama.cpp b/llama.cpp index e91ad7285..adbcc07e2 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4424,7 +4424,9 @@ static void llm_load_vocab( } else if ( tokenizer_pre == "gpt-2" || tokenizer_pre == "jina-es" || - tokenizer_pre == "jina-de") { + tokenizer_pre == "jina-de" || + tokenizer_pre == "jina-v2-es" || + tokenizer_pre == "jina-v2-de") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT2; } else if ( tokenizer_pre == "refact") { From 948f4ec7c5bff92b18e63303f2b2d1645bccd943 Mon Sep 17 00:00:00 2001 From: Neo Zhang <14088817+arthw@users.noreply.github.com> Date: Mon, 13 May 2024 18:11:26 +0800 Subject: [PATCH 007/137] [SYCL] rm wait() (#7233) --- ggml-sycl.cpp | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index e93d2af63..724070eb9 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -15564,26 +15564,6 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, const int64_t r2 = ne12/ne02; const int64_t r3 = ne13/ne03; -#if 0 - // use syclGemmEx - { - for (int i13 = 0; i13 < ne13; ++i13) { - for (int i12 = 0; i12 < ne12; ++i12) { - int i03 = i13 / r3; - int i02 = i12 / r2; - - SYCL_CHECK( - syclGemmEx(g_sycl_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N, - ne01, ne11, ne10, - alpha, (const char *) src0_as_f16 + i02*src0->nb[2] + i03*src0->nb[3] , SYCL_R_16F, nb01/sizeof(half), - (const char *) src1_as_f16 + i12*src1->nb[2]/2 + i13*src1->nb[3]/2, SYCL_R_16F, nb11/sizeof(float), - beta, ( char *) dst_t + i12*nbd2 + i13*nbd3, cu_data_type, ne01, - cu_compute_type, - CUBLAS_GEMM_DEFAULT_TENSOR_OP)); - } - } - } -#else if (r2 == 1 && r3 == 1 && src0->nb[2]*src0->ne[2] == src0->nb[3] && src1->nb[2]*src1->ne[2] == src1->nb[3]) { // there is no broadcast and src0, src1 are contiguous across dims 2, 3 SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm_batch( @@ -15595,7 +15575,6 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, nb11 / nb10, nb12 / nb10, beta, (char *)dst_t, cu_data_type, ne01, nb2 / nb0, ne12 * ne13, cu_compute_type))); - g_sycl_handles[g_main_device]->wait(); } else { const int ne23 = ne12*ne13; @@ -15626,7 +15605,7 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, nb02, nb03, nb12_scaled, nb13_scaled, nbd2, nbd3, r2, r3, item_ct1); }); - }).wait(); + }); } SYCL_CHECK(CHECK_TRY_ERROR(dpct::gemm_batch( *g_sycl_handles[g_main_device], oneapi::mkl::transpose::trans, @@ -15637,9 +15616,7 @@ static void ggml_sycl_mul_mat_batched_sycl(const ggml_tensor *src0, dpct::library_data_t::real_half, nb11 / nb10, beta, (void **)(ptrs_dst.get() + 0 * ne23), cu_data_type, ne01, ne23, cu_compute_type))); - g_sycl_handles[g_main_device]->wait(); } -#endif if (no_mixed_dtypes) { const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16); From 1c570d8beeebad95872dc738ea542a4a0022f78a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Mon, 13 May 2024 13:03:27 +0200 Subject: [PATCH 008/137] perplexity: add BF16 vs. FP16 results (#7150) --- examples/perplexity/README.md | 60 ++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/examples/perplexity/README.md b/examples/perplexity/README.md index c5e2bc5de..c2a3c5ce9 100644 --- a/examples/perplexity/README.md +++ b/examples/perplexity/README.md @@ -7,6 +7,8 @@ Also note that finetunes typically result in a higher perplexity value even thou Within llama.cpp the perplexity of base models is used primarily to judge the quality loss from e.g. quantized models vs. FP16. The convention among contributors is to use the Wikitext-2 test set for testing unless noted otherwise (can be obtained with `scripts/get-wikitext-2.sh`). +When numbers are listed all command line arguments and compilation options are left at their defaults unless noted otherwise. +llama.cpp numbers are **not** directly comparable to those of other projects because the exact values depend strongly on the implementation details. By default only the mean perplexity value and the corresponding uncertainty is calculated. The uncertainty is determined empirically by assuming a Gaussian distribution of the "correct" logits per and then applying error propagation. @@ -32,7 +34,13 @@ In addition to the KL divergence the following statistics are calculated with `- ## LLaMA 3 8b Scoreboard -Results are sorted by Kullback-Leibler divergence relative to FP16. +| Revision | f364eb6f | +|:---------|:-------------------| +| Backend | CUDA | +| CPU | AMD Epyc 7742 | +| GPU | 1x NVIDIA RTX 4090 | + +Results were generated using the CUDA backend and are sorted by Kullback-Leibler divergence relative to FP16. The "WT" importance matrices were created using varying numbers of Wikitext tokens and can be found [here](https://huggingface.co/JohannesGaessler/llama.cpp_importance_matrices/blob/main/imatrix-llama_3-8b-f16-2.7m_tokens.dat). | Quantization | imatrix | Model size [GiB] | PPL | ΔPPL | KLD | Mean Δp | RMS Δp | @@ -89,6 +97,12 @@ K-quants score better on mean Δp than the legacy quants than e.g. KL divergence ## LLaMA 2 vs. LLaMA 3 Quantization comparison +| Revision | f364eb6f | +|:---------|:-------------------| +| Backend | CUDA | +| CPU | AMD Epyc 7742 | +| GPU | 1x NVIDIA RTX 4090 | + | Metric | L2 7b q2_K | L3 8b q2_K | L2 7b q4_K_M | L3 8b q4_K_M | L2 7b q6_K | L3 8b q6_K | L2 7b q8_0 | L3 8b q8_0 | |-----------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------|---------------------| | Mean PPL | 5.794552 ± 0.032298 | 9.751568 ± 0.063312 | 5.877078 ± 0.032781 | 6.407115 ± 0.039119 | 5.808494 ± 0.032425 | 6.253382 ± 0.038078 | 5.798542 ± 0.032366 | 6.234284 ± 0.037878 | @@ -107,6 +121,50 @@ K-quants score better on mean Δp than the legacy quants than e.g. KL divergence | RMS Δp | 9.762 ± 0.053 % | 21.421 ± 0.079 % | 3.252 ± 0.024 % | 5.519 ± 0.050 % | 1.339 ± 0.010 % | 2.295 ± 0.019 % | 0.618 ± 0.011 % | 1.198 ± 0.007 % | | Same top p | 85.584 ± 0.086 % | 71.138 ± 0.119 % | 94.665 ± 0.055 % | 91.901 ± 0.072 % | 97.520 ± 0.038 % | 96.031 ± 0.051 % | 98.846 ± 0.026 % | 97.674 ± 0.040 % | +## LLaMA 3 BF16 vs. FP16 comparison + +| Revision | 83330d8c | +|:---------|:--------------| +| Backend | CPU | +| CPU | AMD Epyc 7742 | +| GPU | N/A | + +Results were calculated with LLaMA 3 8b BF16 as `--kl-divergence-base` and LLaMA 3 8b FP16 as the `--model` for comparison. + +| Metric | Value | +|--------------------------------|--------------------------| +| Mean PPL(Q) | 6.227711 ± 0.037833 | +| Mean PPL(base) | 6.225194 ± 0.037771 | +| Cor(ln(PPL(Q)), ln(PPL(base))) | 99.990% | +| Mean ln(PPL(Q)/PPL(base)) | 0.000404 ± 0.000086 | +| Mean PPL(Q)/PPL(base) | 1.000404 ± 0.000086 | +| Mean PPL(Q)-PPL(base) | 0.002517 ± 0.000536 | +| Mean KLD | 0.00002515 ± 0.00000020 | +| Maximum KLD | 0.012206 | +| 99.9% KLD | 0.000799 | +| 99.0% KLD | 0.000222 | +| 99.0% KLD | 0.000222 | +| Median KLD | 0.000013 | +| 10.0% KLD | -0.000002 | +| 5.0% KLD | -0.000008 | +| 1.0% KLD | -0.000023 | +| Minimum KLD | -0.000059 | +| Mean Δp | -0.0000745 ± 0.0003952 % | +| Maximum Δp | 4.186% | +| 99.9% Δp | 1.049% | +| 99.0% Δp | 0.439% | +| 95.0% Δp | 0.207% | +| 90.0% Δp | 0.125% | +| 75.0% Δp | 0.029% | +| Median Δp | 0.000% | +| 25.0% Δp | -0.030% | +| 10.0% Δp | -0.126% | +| 5.0% Δp | -0.207% | +| 1.0% Δp | -0.434% | +| 0.1% Δp | -1.016% | +| Minimum Δp | -4.672% | +| RMS Δp | 0.150 ± 0.001 % | +| Same top p | 99.739 ± 0.013 % | ## Old Numbers From 30e70334f71b3bd115024affcf98cac3d79aaa95 Mon Sep 17 00:00:00 2001 From: "k.h.lai" Date: Mon, 13 May 2024 22:02:36 +0800 Subject: [PATCH 009/137] llava-cli: fix base64 prompt (#7248) --- examples/llava/llava-cli.cpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/examples/llava/llava-cli.cpp b/examples/llava/llava-cli.cpp index da60ddf2f..a6d67e5d7 100644 --- a/examples/llava/llava-cli.cpp +++ b/examples/llava/llava-cli.cpp @@ -300,14 +300,10 @@ int main(int argc, char ** argv) { return 1; } - for (auto & image : params.image) { + if (prompt_contains_image(params.prompt)) { auto ctx_llava = llava_init_context(¶ms, model); - auto image_embed = load_image(ctx_llava, ¶ms, image); - if (!image_embed) { - std::cerr << "error: failed to load image " << image << ". Terminating\n\n"; - return 1; - } + auto image_embed = load_image(ctx_llava, ¶ms, ""); // process the prompt process_prompt(ctx_llava, image_embed, ¶ms, params.prompt); @@ -316,7 +312,26 @@ int main(int argc, char ** argv) { llava_image_embed_free(image_embed); ctx_llava->model = NULL; llava_free(ctx_llava); + } else { + for (auto & image : params.image) { + auto ctx_llava = llava_init_context(¶ms, model); + + auto image_embed = load_image(ctx_llava, ¶ms, image); + if (!image_embed) { + std::cerr << "error: failed to load image " << image << ". Terminating\n\n"; + return 1; + } + + // process the prompt + process_prompt(ctx_llava, image_embed, ¶ms, params.prompt); + + llama_print_timings(ctx_llava->ctx_llama); + llava_image_embed_free(image_embed); + ctx_llava->model = NULL; + llava_free(ctx_llava); + } } + llama_free_model(model); return 0; From 614d3b914e1c3e02596f869649eb4f1d3b68614d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 13 May 2024 17:15:15 +0300 Subject: [PATCH 010/137] llama : less KV padding when FA is off (#7257) ggml-ci --- llama.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/llama.cpp b/llama.cpp index adbcc07e2..202bf94c8 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2805,6 +2805,11 @@ static void llama_kv_cache_defrag(struct llama_kv_cache & cache) { cache.do_defrag = true; } +static uint32_t llama_kv_cache_get_padding(const struct llama_cparams & cparams) { + // the FA kernels require padding to avoid extra runtime boundary checks + return cparams.flash_attn ? 256u : 32u; +} + // // model loading and saving // @@ -11510,7 +11515,8 @@ static int llama_decode_internal( // a heuristic, to avoid attending the full cache if it is not yet utilized // after enough generations, the benefit from this heuristic disappears // if we start defragmenting the cache, the benefit from this will be more important - kv_self.n = std::min(kv_self.size, std::max(256u, GGML_PAD(llama_kv_cache_cell_max(kv_self), 256))); + const uint32_t pad = llama_kv_cache_get_padding(cparams); + kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad))); //kv_self.n = llama_kv_cache_cell_max(kv_self); } } @@ -15511,6 +15517,11 @@ struct llama_context * llama_new_context_with_model( return nullptr; } + if (params.flash_attn && model->arch == LLM_ARCH_GROK) { + LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__); + params.flash_attn = false; + } + llama_context * ctx = new llama_context(*model); const auto & hparams = model->hparams; @@ -15534,7 +15545,7 @@ struct llama_context * llama_new_context_with_model( cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale; // this is necessary due to kv_self.n being padded later during inference - cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256); + cparams.n_ctx = GGML_PAD(cparams.n_ctx, llama_kv_cache_get_padding(cparams)); // with causal attention, the batch size is limited by the context size cparams.n_batch = hparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch; @@ -15579,11 +15590,6 @@ struct llama_context * llama_new_context_with_model( } } - if (cparams.flash_attn && model->arch == LLM_ARCH_GROK) { - LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__); - cparams.flash_attn = false; - } - if (params.seed == LLAMA_DEFAULT_SEED) { params.seed = time(NULL); } From ee52225067622babc277371511b8124884e1c797 Mon Sep 17 00:00:00 2001 From: compilade Date: Mon, 13 May 2024 14:10:51 -0400 Subject: [PATCH 011/137] convert-hf : support direct Q8_0 conversion (#7234) * convert-hf : support q8_0 conversion * convert-hf : add missing ftype This was messing with the checksums otherwise. * convert-hf : add missing ftype to Baichuan and Xverse I didn't notice these on my first pass. --- convert-hf-to-gguf.py | 72 +++++++++--------------- gguf-py/gguf/__init__.py | 1 + gguf-py/gguf/gguf_writer.py | 16 ++++-- gguf-py/gguf/lazy.py | 29 +++++++--- gguf-py/gguf/quants.py | 109 ++++++++++++++++++++++++++++++++++++ 5 files changed, 169 insertions(+), 58 deletions(-) create mode 100644 gguf-py/gguf/quants.py diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index d6e5dece0..cd875fa4a 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -240,23 +240,6 @@ class Model: return False def write_tensors(self): - # same as ggml_compute_fp32_to_bf16 in ggml-impl.h - def np_fp32_to_bf16(n: np.ndarray): - # force nan to quiet - n = np.where((n & 0x7fffffff) > 0x7f800000, (n & 0xffff0000) | (64 << 16), n) - # flush subnormals to zero - n = np.where((n & 0x7f800000) == 0, n & 0x80000000, n) - # round to nearest even - n = (n + (0x7fff + ((n >> 16) & 1))) >> 16 - return n.astype(np.int16) - - # Doing this row-wise is much, much faster than element-wise, hence the signature - v_fp32_to_bf16 = np.vectorize(np_fp32_to_bf16, otypes=[np.int16], signature="(n)->(n)") - if self.lazy: - # TODO: find a way to implicitly wrap np.vectorize functions - # NOTE: the type is changed to reflect otypes passed to np.vectorize above - v_fp32_to_bf16 = gguf.LazyNumpyTensor._wrap_fn(v_fp32_to_bf16, meta_noop=np.int16) - max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,") for name, data_torch in self.get_tensors(): @@ -309,27 +292,31 @@ class Model: )) if self.ftype != gguf.LlamaFileType.ALL_F32 and extra_f16 and not extra_f32: - if self.ftype == gguf.LlamaFileType.MOSTLY_F16: + if self.ftype == gguf.LlamaFileType.MOSTLY_BF16: + data = gguf.quantize_bf16(data) + assert data.dtype == np.int16 + data_qtype = gguf.GGMLQuantizationType.BF16 + + elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0 and gguf.can_quantize_to_q8_0(data): + data = gguf.quantize_q8_0(data) + assert data.dtype == np.uint8 + data_qtype = gguf.GGMLQuantizationType.Q8_0 + + else: # default to float16 for quantized tensors if data_dtype != np.float16: data = data.astype(np.float16) data_qtype = gguf.GGMLQuantizationType.F16 - elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16: - if data_dtype != np.float32: - data = data.astype(np.float32) - data = v_fp32_to_bf16(data.view(np.int32)) - assert data.dtype == np.int16 - data_qtype = gguf.GGMLQuantizationType.BF16 - - else: # by default, convert to float32 + if data_qtype is None: # by default, convert to float32 if data_dtype != np.float32: data = data.astype(np.float32) data_qtype = gguf.GGMLQuantizationType.F32 - assert data_qtype is not None - + block_size, type_size = gguf.GGML_QUANT_SIZES[data_qtype] # reverse shape to make it similar to the internal ggml dimension order - shape_str = f"{{{', '.join(str(n) for n in reversed(data.shape))}}}" + shape_str = f"""{{{', '.join(str(n) for n in reversed( + (*data.shape[:-1], data.shape[-1] * data.dtype.itemsize // type_size * block_size)) + )}}}""" # n_dims is implicit in the shape logger.info(f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}") @@ -859,6 +846,7 @@ class BaichuanModel(Model): self.gguf_writer.add_head_count(head_count) self.gguf_writer.add_head_count_kv(head_count_kv) self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) + self.gguf_writer.add_file_type(self.ftype) if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]: if self.hparams["rope_scaling"].get("type") == "linear": @@ -981,6 +969,7 @@ class XverseModel(Model): self.gguf_writer.add_head_count(head_count) self.gguf_writer.add_head_count_kv(head_count_kv) self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) + self.gguf_writer.add_file_type(self.ftype) if self.hparams.get("rope_scaling") is not None and "factor" in self.hparams["rope_scaling"]: if self.hparams["rope_scaling"].get("type") == "linear": @@ -1215,6 +1204,7 @@ class StableLMModel(Model): self.gguf_writer.add_head_count_kv(hparams["num_key_value_heads"]) self.gguf_writer.add_parallel_residual(hparams["use_parallel_residual"] if "use_parallel_residual" in hparams else True) self.gguf_writer.add_layer_norm_eps(self.find_hparam(["layer_norm_eps", "norm_eps"])) + self.gguf_writer.add_file_type(self.ftype) _q_norms: list[dict[str, Tensor]] | None = None _k_norms: list[dict[str, Tensor]] | None = None @@ -1591,6 +1581,7 @@ class QwenModel(Model): self.gguf_writer.add_rope_dimension_count(self.hparams["hidden_size"] // self.hparams["num_attention_heads"]) self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) self.gguf_writer.add_layer_norm_rms_eps(self.hparams["layer_norm_epsilon"]) + self.gguf_writer.add_file_type(self.ftype) @Model.register("Qwen2ForCausalLM") @@ -1828,6 +1819,7 @@ class PlamoModel(Model): self.gguf_writer.add_head_count(hparams["num_attention_heads"]) self.gguf_writer.add_head_count_kv(5) # hparams["num_key_value_heads"]) is wrong self.gguf_writer.add_layer_norm_rms_eps(hparams["rms_norm_eps"]) + self.gguf_writer.add_file_type(self.ftype) def shuffle_attn_q_weight(self, data_torch): assert data_torch.size() == (5120, 5120) @@ -2007,6 +1999,7 @@ in chat mode so that the conversation can end normally.") self.gguf_writer.add_head_count(self.hparams["num_attention_heads"]) self.gguf_writer.add_layer_norm_rms_eps(self.hparams["rms_norm_eps"]) self.gguf_writer.add_head_count_kv(self.hparams["num_key_value_heads"]) + self.gguf_writer.add_file_type(self.ftype) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: num_heads = self.hparams["num_attention_heads"] @@ -2415,25 +2408,15 @@ class LazyTorchTensor(gguf.LazyBase): def numpy(self) -> gguf.LazyNumpyTensor: dtype = self._dtype_map[self.dtype] return gguf.LazyNumpyTensor( - meta=np.lib.stride_tricks.as_strided(np.zeros(1, dtype), self.shape, (0 for _ in self.shape)), + meta=gguf.LazyNumpyTensor.meta_with_dtype_and_shape(dtype, self.shape), lazy=self._lazy, args=(self,), func=(lambda s: s[0].numpy()) ) @classmethod - def eager_to_meta(cls, t: Tensor) -> Tensor: - if t.is_meta: - return t - return t.detach().to("meta") - - @classmethod - def meta_with_dtype(cls, m: Tensor, dtype: torch.dtype) -> Tensor: - m = m.detach() - if not m.is_meta: - m = m.to("meta") - m.dtype = dtype - return m + def meta_with_dtype_and_shape(cls, dtype: torch.dtype, shape: torch.Size) -> Tensor: + return torch.empty(size=shape, dtype=dtype, device="meta") @classmethod def __torch_function__(cls, func, types, args=(), kwargs=None): @@ -2464,8 +2447,8 @@ def parse_args() -> argparse.Namespace: help="path to write to; default: based on input. {ftype} will be replaced by the outtype.", ) parser.add_argument( - "--outtype", type=str, choices=["f32", "f16", "bf16", "auto"], default="f16", - help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, auto for the highest-fidelity 16-bit float type depending on the first loaded tensor type", + "--outtype", type=str, choices=["f32", "f16", "bf16", "q8_0", "auto"], default="f16", + help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, auto for the highest-fidelity 16-bit float type depending on the first loaded tensor type", ) parser.add_argument( "--bigendian", action="store_true", @@ -2523,6 +2506,7 @@ def main() -> None: "f32": gguf.LlamaFileType.ALL_F32, "f16": gguf.LlamaFileType.MOSTLY_F16, "bf16": gguf.LlamaFileType.MOSTLY_BF16, + "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0, "auto": gguf.LlamaFileType.GUESSED, } diff --git a/gguf-py/gguf/__init__.py b/gguf-py/gguf/__init__.py index e5d5806c8..ea5146b16 100644 --- a/gguf-py/gguf/__init__.py +++ b/gguf-py/gguf/__init__.py @@ -2,5 +2,6 @@ from .constants import * from .lazy import * from .gguf_reader import * from .gguf_writer import * +from .quants import * from .tensor_mapping import * from .vocab import * diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 96574358d..d5e323a52 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -13,6 +13,7 @@ from string import ascii_letters, digits import numpy as np from .constants import ( + GGML_QUANT_SIZES, GGUF_DEFAULT_ALIGNMENT, GGUF_MAGIC, GGUF_VERSION, @@ -195,7 +196,7 @@ class GGUFWriter: return ((x + n - 1) // n) * n def add_tensor_info( - self, name: str, tensor_shape: Sequence[int], tensor_dtype: np.dtype[np.float16] | np.dtype[np.float32], + self, name: str, tensor_shape: Sequence[int], tensor_dtype: np.dtype, tensor_nbytes: int, raw_dtype: GGMLQuantizationType | None = None, ) -> None: if self.state is not WriterState.EMPTY: @@ -208,10 +209,6 @@ class GGUFWriter: encoded_name = name.encode("utf-8") self.ti_data += self._pack("Q", len(encoded_name)) self.ti_data += encoded_name - n_dims = len(tensor_shape) - self.ti_data += self._pack("I", n_dims) - for i in range(n_dims): - self.ti_data += self._pack("Q", tensor_shape[n_dims - 1 - i]) if raw_dtype is None: if tensor_dtype == np.float16: dtype = GGMLQuantizationType.F16 @@ -231,6 +228,15 @@ class GGUFWriter: raise ValueError("Only F16, F32, F64, I8, I16, I32, I64 tensors are supported for now") else: dtype = raw_dtype + if tensor_dtype == np.uint8: + block_size, type_size = GGML_QUANT_SIZES[raw_dtype] + if tensor_shape[-1] % type_size != 0: + raise ValueError(f"Quantized tensor row size ({tensor_shape[-1]}) is not a multiple of {dtype.name} type size ({type_size})") + tensor_shape = tuple(tensor_shape[:-1]) + (tensor_shape[-1] // type_size * block_size,) + n_dims = len(tensor_shape) + self.ti_data += self._pack("I", n_dims) + for i in range(n_dims): + self.ti_data += self._pack("Q", tensor_shape[n_dims - 1 - i]) self.ti_data += self._pack("I", dtype) self.ti_data += self._pack("Q", self.offset_tensor) self.offset_tensor += GGUFWriter.ggml_pad(tensor_nbytes, self.data_alignment) diff --git a/gguf-py/gguf/lazy.py b/gguf-py/gguf/lazy.py index 650bea11c..1167335b8 100644 --- a/gguf-py/gguf/lazy.py +++ b/gguf-py/gguf/lazy.py @@ -6,6 +6,7 @@ from typing import Any, Callable from collections import deque import numpy as np +from numpy._typing import _Shape from numpy.typing import DTypeLike @@ -110,7 +111,7 @@ class LazyBase(ABC, metaclass=LazyMeta): return o @classmethod - def _wrap_fn(cls, fn: Callable, *, use_self: LazyBase | None = None, meta_noop: bool | DTypeLike = False) -> Callable[[Any], Any]: + def _wrap_fn(cls, fn: Callable, *, use_self: LazyBase | None = None, meta_noop: bool | DTypeLike | tuple[DTypeLike, Callable[[tuple[int, ...]], tuple[int, ...]]] = False) -> Callable[[Any], Any]: def wrapped_fn(*args, **kwargs): if kwargs is None: kwargs = {} @@ -130,9 +131,14 @@ class LazyBase(ABC, metaclass=LazyMeta): res = args[0] assert isinstance(res, cls) res = res._meta - # allow operations to override the dtype + # allow operations to override the dtype and shape if meta_noop is not True: - res = cls.meta_with_dtype(res, meta_noop) + if isinstance(meta_noop, tuple): + dtype, shape = meta_noop + assert callable(shape) + res = cls.meta_with_dtype_and_shape(dtype, shape(res.shape)) + else: + res = cls.meta_with_dtype_and_shape(meta_noop, res.shape) if isinstance(res, cls._tensor_type): def collect_replace(t: LazyBase): @@ -168,7 +174,12 @@ class LazyBase(ABC, metaclass=LazyMeta): while _t._data is None: lt = _t._lazy.popleft() if lt._data is not None: - raise ValueError(f"{lt} did not belong in the lazy queue") + # Lazy tensor did not belong in the lazy queue. + # Weirdly only happens with Bloom models... + # likely because tensors aren't unique in the queue. + # The final output is still the same as in eager mode, + # so it's safe to ignore this. + continue assert lt._func is not None lt._args = cls._recurse_apply(lt._args, already_eager_to_eager) lt._data = lt._func(lt._args) @@ -183,12 +194,12 @@ class LazyBase(ABC, metaclass=LazyMeta): @classmethod def eager_to_meta(cls, t: Any) -> Any: - return cls.meta_with_dtype(t, t.dtype) + return cls.meta_with_dtype_and_shape(t.dtype, t.shape) # must be overridden, meta tensor init is backend-specific @classmethod @abstractmethod - def meta_with_dtype(cls, m: Any, dtype: Any) -> Any: pass + def meta_with_dtype_and_shape(cls, dtype: Any, shape: Any) -> Any: pass @classmethod def from_eager(cls, t: Any) -> Any: @@ -205,15 +216,15 @@ class LazyNumpyTensor(LazyBase): _tensor_type = np.ndarray @classmethod - def meta_with_dtype(cls, m: np.ndarray[Any, Any], dtype: DTypeLike) -> np.ndarray[Any, Any]: + def meta_with_dtype_and_shape(cls, dtype: DTypeLike, shape: _Shape) -> np.ndarray[Any, Any]: # The initial idea was to use np.nan as the fill value, # but non-float types like np.int16 can't use that. # So zero it is. cheat = np.zeros(1, dtype) - return np.lib.stride_tricks.as_strided(cheat, m.shape, (0 for _ in m.shape)) + return np.lib.stride_tricks.as_strided(cheat, shape, (0 for _ in shape)) def astype(self, dtype, *args, **kwargs): - meta = type(self).meta_with_dtype(self._meta, dtype) + meta = type(self).meta_with_dtype_and_shape(dtype, self._meta.shape) full_args = (self, dtype,) + args # very important to pass the shared _lazy deque, or else there's an infinite loop somewhere. return type(self)(meta=meta, args=full_args, lazy=self._lazy, func=(lambda a: a[0].astype(*a[1:], **kwargs))) diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py new file mode 100644 index 000000000..e7fc0eae3 --- /dev/null +++ b/gguf-py/gguf/quants.py @@ -0,0 +1,109 @@ +from __future__ import annotations +from typing import Callable + +from numpy.typing import DTypeLike + +from .constants import GGML_QUANT_SIZES, GGMLQuantizationType +from .lazy import LazyNumpyTensor + +import numpy as np + + +# same as ggml_compute_fp32_to_bf16 in ggml-impl.h +def __compute_fp32_to_bf16(n: np.ndarray) -> np.ndarray: + n = n.astype(np.float32, copy=False).view(np.int32) + # force nan to quiet + n = np.where((n & 0x7fffffff) > 0x7f800000, (n & 0xffff0000) | (64 << 16), n) + # flush subnormals to zero + n = np.where((n & 0x7f800000) == 0, n & 0x80000000, n) + # round to nearest even + n = (n + (0x7fff + ((n >> 16) & 1))) >> 16 + return n.astype(np.int16) + + +# This is faster than np.vectorize and np.apply_along_axis because it works on more than one row at a time +def __apply_over_grouped_rows(func: Callable[[np.ndarray], np.ndarray], arr: np.ndarray, otype: DTypeLike, oshape: tuple[int, ...]) -> np.ndarray: + rows = arr.reshape((-1, arr.shape[-1])) + osize = 1 + for dim in oshape: + osize *= dim + out = np.empty(shape=osize, dtype=otype) + # compute over groups of 16 rows (arbitrary, but seems good for performance) + n_groups = rows.shape[0] // 16 + np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out) + return out.reshape(oshape) + + +def __quantize_bf16_array(n: np.ndarray) -> np.ndarray: + return __apply_over_grouped_rows(__compute_fp32_to_bf16, arr=n, otype=np.int16, oshape=n.shape) + + +__quantize_bf16_lazy = LazyNumpyTensor._wrap_fn(__quantize_bf16_array, meta_noop=np.int16) + + +def quantize_bf16(n: np.ndarray): + if type(n) is LazyNumpyTensor: + return __quantize_bf16_lazy(n) + else: + return __quantize_bf16_array(n) + + +__q8_block_size, __q8_type_size = GGML_QUANT_SIZES[GGMLQuantizationType.Q8_0] + + +def can_quantize_to_q8_0(n: np.ndarray) -> bool: + return n.shape[-1] % __q8_block_size == 0 + + +# round away from zero +# ref: https://stackoverflow.com/a/59143326/22827863 +def np_roundf(n: np.ndarray) -> np.ndarray: + a = abs(n) + floored = np.floor(a) + b = floored + np.floor(2 * (a - floored)) + return np.sign(n) * b + + +def __quantize_q8_0_shape_change(s: tuple[int, ...]) -> tuple[int, ...]: + return (*s[:-1], s[-1] // __q8_block_size * __q8_type_size) + + +# Implementation of Q8_0 with bit-exact same results as reference implementation in ggml-quants.c +def __quantize_q8_0_rows(n: np.ndarray) -> np.ndarray: + shape = n.shape + assert shape[-1] % __q8_block_size == 0 + + n_blocks = n.size // __q8_block_size + + blocks = n.reshape((n_blocks, __q8_block_size)).astype(np.float32, copy=False) + + d = abs(blocks).max(axis=1, keepdims=True) / 127 + with np.errstate(divide="ignore"): + id = np.where(d == 0, 0, 1 / d) + qs = np_roundf(blocks * id) + + # (n_blocks, 2) + d = d.astype(np.float16).view(np.uint8) + # (n_blocks, block_size) + qs = qs.astype(np.int8).view(np.uint8) + + assert d.shape[1] + qs.shape[1] == __q8_type_size + + return np.concatenate([d, qs], axis=1).reshape(__quantize_q8_0_shape_change(shape)) + + +def __quantize_q8_0_array(n: np.ndarray) -> np.ndarray: + return __apply_over_grouped_rows(__quantize_q8_0_rows, arr=n, otype=np.uint8, oshape=__quantize_q8_0_shape_change(n.shape)) + + +__quantize_q8_0_lazy = LazyNumpyTensor._wrap_fn( + __quantize_q8_0_array, + meta_noop=(np.uint8, __quantize_q8_0_shape_change), +) + + +def quantize_q8_0(data: np.ndarray): + if type(data) is LazyNumpyTensor: + return __quantize_q8_0_lazy(data) + else: + return __quantize_q8_0_array(data) From 27f65d6267cf22a44c5ccefa7765d53a05bd1259 Mon Sep 17 00:00:00 2001 From: Ryuei Date: Tue, 14 May 2024 14:20:47 +0900 Subject: [PATCH 012/137] docs: Fix typo and update description for --embeddings flag (#7026) - Change '--embedding' to '--embeddings' in the README - Update the description to match the latest --help output - Added a caution about defining physical batch size --- examples/server/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/server/README.md b/examples/server/README.md index 650317991..f6eb6942c 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -48,7 +48,7 @@ page cache before using this. See https://github.com/ggerganov/llama.cpp/issues/ - `--path`: Path from which to serve static files. Default: disabled - `--api-key`: Set an api key for request authorization. By default, the server responds to every request. With an api key set, the requests must have the Authorization header set with the api key as Bearer token. May be used multiple times to enable multiple valid keys. - `--api-key-file`: Path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access. May be used in conjunction with `--api-key`s. -- `--embedding`: Enable embedding extraction. Default: disabled +- `--embeddings`: Enable embedding vector output and the OAI compatible endpoint /v1/embeddings. Physical batch size (`--ubatch-size`) must be carefully defined. Default: disabled - `-np N`, `--parallel N`: Set the number of slots for process requests. Default: `1` - `-cb`, `--cont-batching`: Enable continuous batching (a.k.a dynamic batching). Default: disabled - `-spf FNAME`, `--system-prompt-file FNAME` Set a file to load a system prompt (initial prompt of all slots). This is useful for chat applications. [See more](#change-system-prompt-on-runtime) From e0f556186b6e1f2b7032a1479edf5e89e2b1bd86 Mon Sep 17 00:00:00 2001 From: Haggai Nuchi Date: Mon, 13 May 2024 22:25:56 -0700 Subject: [PATCH 013/137] Add left recursion check: quit early instead of going into an infinite loop (#7083) * Add left recursion check: quit early instead of going into an infinite loop * Remove custom enum, rename left recursion check and move to "grammar internal" section, add handling for edge case where a leftmost nonterminal may be empty * Remove unnecessary declaration --- llama.cpp | 68 ++++++++++++++++++++++++++++++ tests/test-grammar-integration.cpp | 46 ++++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/llama.cpp b/llama.cpp index 202bf94c8..01a35dfb6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13182,6 +13182,58 @@ static std::vector llama_grammar_reject_candidates( return rejects; } +static bool llama_grammar_detect_left_recursion( + const std::vector> & rules, + size_t rule_index, + std::vector * rules_visited, + std::vector * rules_in_progress, + std::vector * rules_may_be_empty) { + if ((*rules_in_progress)[rule_index]) { + return true; + } + + (*rules_in_progress)[rule_index] = true; + + const std::vector & rule = rules[rule_index]; + + // First check if the rule might produce the empty string. This could be done combined with the second + // step but it's more readable as two steps. + bool at_rule_start = true; + for (size_t i = 0; i < rule.size(); i++) { + if (llama_grammar_is_end_of_sequence(&rule[i])) { + if (at_rule_start) { + (*rules_may_be_empty)[rule_index] = true; + break; + } + at_rule_start = true; + } else { + at_rule_start = false; + } + } + + // Second, recurse into leftmost nonterminals (or next-leftmost as long as the previous nonterminal may + // be empty) + bool recurse_into_nonterminal = true; + for (size_t i = 0; i < rule.size(); i++) { + if (rule[i].type == LLAMA_GRETYPE_RULE_REF && recurse_into_nonterminal) { + if (llama_grammar_detect_left_recursion(rules, (size_t)rule[i].value, rules_visited, rules_in_progress, rules_may_be_empty)) { + return true; + } + if (!((*rules_may_be_empty)[(size_t)rule[i].value])) { + recurse_into_nonterminal = false; + } + } else if (llama_grammar_is_end_of_sequence(&rule[i])) { + recurse_into_nonterminal = true; + } else { + recurse_into_nonterminal = false; + } + } + + (*rules_in_progress)[rule_index] = false; + (*rules_visited)[rule_index] = true; + return false; +} + // // grammar - external // @@ -13201,6 +13253,19 @@ struct llama_grammar * llama_grammar_init( vec_rules[i].push_back({LLAMA_GRETYPE_END, 0}); } + // Check for left recursion + std::vector rules_visited(n_rules); + std::vector rules_in_progress(n_rules); + std::vector rules_may_be_empty(n_rules); + for (size_t i = 0; i < n_rules; i++) { + if (rules_visited[i]) { + continue; + } + if (llama_grammar_detect_left_recursion(vec_rules, i, &rules_visited, &rules_in_progress, &rules_may_be_empty)) { + throw std::runtime_error(format("unsupported grammar, left recursion detected for nonterminal at index %zu", i)); + } + } + // loop over alternates of start rule to build initial stacks std::vector> stacks; pos = vec_rules[start_rule_index].data(); @@ -13223,6 +13288,9 @@ struct llama_grammar * llama_grammar_init( } } while (true); + // Important: vec_rules has to be moved here, not copied, because stacks contains + // pointers to elements of vec_rules. If vec_rules were copied into llama_grammar + // then the pointers would be invalidated when the local vec_rules goes out of scope. return new llama_grammar{ std::move(vec_rules), std::move(stacks), {} }; } diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index 1a4004e2a..01c5bb27a 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -28,6 +28,19 @@ static llama_grammar* build_grammar(const std::string & grammar_str) { return grammar; } +static bool test_build_grammar_fails(const std::string & grammar_str) { + fprintf(stderr, "⚫ Testing failure for grammar: %s\n", grammar_str.c_str()); + bool grammar_fails = false; + try { + build_grammar(grammar_str); + fprintf(stderr, " ❌ Expected build failure, but succeeded\n"); + } catch (const std::exception & err) { + grammar_fails = true; + fprintf(stdout, " ✅︎\n"); + } + return grammar_fails; +} + static bool match_string(const std::string & input, llama_grammar* grammar) { auto decoded = decode_utf8(input, {}); @@ -320,6 +333,38 @@ number ::= [0-9]+)"""; fprintf(stderr, " ✅︎ Passed\n"); } +static void test_failure_left_recursion() { + fprintf(stderr, "⚫ Testing left recursion detection:\n"); + + // Test simple left recursion detection + const std::string simple_str = R"""(root ::= "a" | root "a")"""; + assert(test_build_grammar_fails(simple_str)); + + // Test more complicated left recursion detection + const std::string medium_str = R"""( +root ::= asdf +asdf ::= "a" | asdf "a" +)"""; + assert(test_build_grammar_fails(medium_str)); + + // Test even more complicated left recursion detection + const std::string hard_str = R"""( +root ::= asdf +asdf ::= "a" | foo "b" +foo ::= "c" | asdf "d" | "e")"""; + assert(test_build_grammar_fails(hard_str)); + + // Test yet even more complicated left recursion detection + const std::string hardest_str = R"""( +root ::= asdf +asdf ::= "a" | foo "b" +foo ::= "c" | empty asdf "d" | "e" +empty ::= "blah" | )"""; + assert(test_build_grammar_fails(hardest_str)); + + fprintf(stderr, " ✅︎ Passed\n"); +} + int main() { fprintf(stdout, "Running grammar integration tests...\n"); test_simple_grammar(); @@ -327,6 +372,7 @@ int main() { test_quantifiers(); test_failure_missing_root(); test_failure_missing_reference(); + test_failure_left_recursion(); fprintf(stdout, "All tests passed.\n"); return 0; } From efc8f767c8c8c749a245dd96ad4e2f37c164b54c Mon Sep 17 00:00:00 2001 From: Elton Kola Date: Tue, 14 May 2024 03:30:30 -0400 Subject: [PATCH 014/137] move ndk code to a new library (#6951) --- examples/llama.android/app/build.gradle.kts | 25 +--------- .../java/com/example/llama/MainViewModel.kt | 13 ++--- examples/llama.android/build.gradle.kts | 1 + examples/llama.android/llama/.gitignore | 1 + .../src/main/cpp => llama}/CMakeLists.txt | 2 +- .../llama.android/llama/consumer-rules.pro | 0 .../llama.android/llama/proguard-rules.pro | 21 ++++++++ .../llama/cpp/ExampleInstrumentedTest.kt | 24 +++++++++ .../llama/src/main/AndroidManifest.xml | 4 ++ .../llama/src/main/cpp/CMakeLists.txt | 49 +++++++++++++++++++ .../src/main/cpp/llama-android.cpp | 28 +++++------ .../java/android/llama/cpp/LLamaAndroid.kt} | 8 +-- .../java/android/llama/cpp/ExampleUnitTest.kt | 17 +++++++ examples/llama.android/settings.gradle.kts | 1 + 14 files changed, 145 insertions(+), 49 deletions(-) create mode 100644 examples/llama.android/llama/.gitignore rename examples/llama.android/{app/src/main/cpp => llama}/CMakeLists.txt (98%) create mode 100644 examples/llama.android/llama/consumer-rules.pro create mode 100644 examples/llama.android/llama/proguard-rules.pro create mode 100644 examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt create mode 100644 examples/llama.android/llama/src/main/AndroidManifest.xml create mode 100644 examples/llama.android/llama/src/main/cpp/CMakeLists.txt rename examples/llama.android/{app => llama}/src/main/cpp/llama-android.cpp (92%) rename examples/llama.android/{app/src/main/java/com/example/llama/Llm.kt => llama/src/main/java/android/llama/cpp/LLamaAndroid.kt} (97%) create mode 100644 examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt diff --git a/examples/llama.android/app/build.gradle.kts b/examples/llama.android/app/build.gradle.kts index d42140efe..8d1b37195 100644 --- a/examples/llama.android/app/build.gradle.kts +++ b/examples/llama.android/app/build.gradle.kts @@ -7,8 +7,6 @@ android { namespace = "com.example.llama" compileSdk = 34 - ndkVersion = "26.1.10909125" - defaultConfig { applicationId = "com.example.llama" minSdk = 33 @@ -20,17 +18,6 @@ android { vectorDrawables { useSupportLibrary = true } - ndk { - // Add NDK properties if wanted, e.g. - // abiFilters += listOf("arm64-v8a") - } - externalNativeBuild { - cmake { - arguments += "-DCMAKE_BUILD_TYPE=Release" - cppFlags += listOf() - arguments += listOf() - } - } } buildTypes { @@ -55,17 +42,6 @@ android { composeOptions { kotlinCompilerExtensionVersion = "1.5.1" } - packaging { - resources { - excludes += "/META-INF/{AL2.0,LGPL2.1}" - } - } - externalNativeBuild { - cmake { - path = file("src/main/cpp/CMakeLists.txt") - version = "3.22.1" - } - } } dependencies { @@ -78,6 +54,7 @@ dependencies { implementation("androidx.compose.ui:ui-graphics") implementation("androidx.compose.ui:ui-tooling-preview") implementation("androidx.compose.material3:material3") + implementation(project(":llama")) testImplementation("junit:junit:4.13.2") androidTestImplementation("androidx.test.ext:junit:1.1.5") androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt index be95e2221..45ac29938 100644 --- a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt +++ b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt @@ -1,5 +1,6 @@ package com.example.llama +import android.llama.cpp.LLamaAndroid import android.util.Log import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -9,7 +10,7 @@ import androidx.lifecycle.viewModelScope import kotlinx.coroutines.flow.catch import kotlinx.coroutines.launch -class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { +class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instance()): ViewModel() { companion object { @JvmStatic private val NanosPerSecond = 1_000_000_000.0 @@ -28,7 +29,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { viewModelScope.launch { try { - llm.unload() + llamaAndroid.unload() } catch (exc: IllegalStateException) { messages += exc.message!! } @@ -44,7 +45,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { messages += "" viewModelScope.launch { - llm.send(text) + llamaAndroid.send(text) .catch { Log.e(tag, "send() failed", it) messages += it.message!! @@ -57,7 +58,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { viewModelScope.launch { try { val start = System.nanoTime() - val warmupResult = llm.bench(pp, tg, pl, nr) + val warmupResult = llamaAndroid.bench(pp, tg, pl, nr) val end = System.nanoTime() messages += warmupResult @@ -70,7 +71,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { return@launch } - messages += llm.bench(512, 128, 1, 3) + messages += llamaAndroid.bench(512, 128, 1, 3) } catch (exc: IllegalStateException) { Log.e(tag, "bench() failed", exc) messages += exc.message!! @@ -81,7 +82,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { fun load(pathToModel: String) { viewModelScope.launch { try { - llm.load(pathToModel) + llamaAndroid.load(pathToModel) messages += "Loaded $pathToModel" } catch (exc: IllegalStateException) { Log.e(tag, "load() failed", exc) diff --git a/examples/llama.android/build.gradle.kts b/examples/llama.android/build.gradle.kts index 50ebc8211..acd1ada7d 100644 --- a/examples/llama.android/build.gradle.kts +++ b/examples/llama.android/build.gradle.kts @@ -2,4 +2,5 @@ plugins { id("com.android.application") version "8.2.0" apply false id("org.jetbrains.kotlin.android") version "1.9.0" apply false + id("com.android.library") version "8.2.0" apply false } diff --git a/examples/llama.android/llama/.gitignore b/examples/llama.android/llama/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/examples/llama.android/llama/.gitignore @@ -0,0 +1 @@ +/build diff --git a/examples/llama.android/app/src/main/cpp/CMakeLists.txt b/examples/llama.android/llama/CMakeLists.txt similarity index 98% rename from examples/llama.android/app/src/main/cpp/CMakeLists.txt rename to examples/llama.android/llama/CMakeLists.txt index 85139329a..bb5738ae3 100644 --- a/examples/llama.android/app/src/main/cpp/CMakeLists.txt +++ b/examples/llama.android/llama/CMakeLists.txt @@ -37,7 +37,7 @@ FetchContent_MakeAvailable(llama) # used in the AndroidManifest.xml file. add_library(${CMAKE_PROJECT_NAME} SHARED # List C/C++ source files with relative paths to this CMakeLists.txt. - llama-android.cpp) + llama-android.cpp) # Specifies libraries CMake should link to your target library. You # can link libraries from various origins, such as libraries defined in this diff --git a/examples/llama.android/llama/consumer-rules.pro b/examples/llama.android/llama/consumer-rules.pro new file mode 100644 index 000000000..e69de29bb diff --git a/examples/llama.android/llama/proguard-rules.pro b/examples/llama.android/llama/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/examples/llama.android/llama/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt b/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt new file mode 100644 index 000000000..05d6ab5d2 --- /dev/null +++ b/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package android.llama.cpp + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("android.llama.cpp.test", appContext.packageName) + } +} diff --git a/examples/llama.android/llama/src/main/AndroidManifest.xml b/examples/llama.android/llama/src/main/AndroidManifest.xml new file mode 100644 index 000000000..8bdb7e14b --- /dev/null +++ b/examples/llama.android/llama/src/main/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + diff --git a/examples/llama.android/llama/src/main/cpp/CMakeLists.txt b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..42ebaad49 --- /dev/null +++ b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt @@ -0,0 +1,49 @@ +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html. +# For more examples on how to use CMake, see https://github.com/android/ndk-samples. + +# Sets the minimum CMake version required for this project. +cmake_minimum_required(VERSION 3.22.1) + +# Declares the project name. The project name can be accessed via ${ PROJECT_NAME}, +# Since this is the top level CMakeLists.txt, the project name is also accessible +# with ${CMAKE_PROJECT_NAME} (both CMake variables are in-sync within the top level +# build script scope). +project("llama-android") + +include(FetchContent) +FetchContent_Declare( + llama + GIT_REPOSITORY https://github.com/ggerganov/llama.cpp + GIT_TAG master +) + +# Also provides "common" +FetchContent_MakeAvailable(llama) + +# Creates and names a library, sets it as either STATIC +# or SHARED, and provides the relative paths to its source code. +# You can define multiple libraries, and CMake builds them for you. +# Gradle automatically packages shared libraries with your APK. +# +# In this top level CMakeLists.txt, ${CMAKE_PROJECT_NAME} is used to define +# the target library name; in the sub-module's CMakeLists.txt, ${PROJECT_NAME} +# is preferred for the same purpose. +# +# In order to load a library into your app from Java/Kotlin, you must call +# System.loadLibrary() and pass the name of the library defined here; +# for GameActivity/NativeActivity derived applications, the same library name must be +# used in the AndroidManifest.xml file. +add_library(${CMAKE_PROJECT_NAME} SHARED + # List C/C++ source files with relative paths to this CMakeLists.txt. + llama-android.cpp) + +# Specifies libraries CMake should link to your target library. You +# can link libraries from various origins, such as libraries defined in this +# build script, prebuilt third-party libraries, or Android system libraries. +target_link_libraries(${CMAKE_PROJECT_NAME} + # List libraries link to the target library + llama + common + android + log) diff --git a/examples/llama.android/app/src/main/cpp/llama-android.cpp b/examples/llama.android/llama/src/main/cpp/llama-android.cpp similarity index 92% rename from examples/llama.android/app/src/main/cpp/llama-android.cpp rename to examples/llama.android/llama/src/main/cpp/llama-android.cpp index 4af9de303..874158ef0 100644 --- a/examples/llama.android/app/src/main/cpp/llama-android.cpp +++ b/examples/llama.android/llama/src/main/cpp/llama-android.cpp @@ -81,7 +81,7 @@ static void log_callback(ggml_log_level level, const char * fmt, void * data) { extern "C" JNIEXPORT jlong JNICALL -Java_com_example_llama_Llm_load_1model(JNIEnv *env, jobject, jstring filename) { +Java_android_llama_cpp_LLamaAndroid_load_1model(JNIEnv *env, jobject, jstring filename) { llama_model_params model_params = llama_model_default_params(); auto path_to_model = env->GetStringUTFChars(filename, 0); @@ -101,13 +101,13 @@ Java_com_example_llama_Llm_load_1model(JNIEnv *env, jobject, jstring filename) { extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_free_1model(JNIEnv *, jobject, jlong model) { +Java_android_llama_cpp_LLamaAndroid_free_1model(JNIEnv *, jobject, jlong model) { llama_free_model(reinterpret_cast(model)); } extern "C" JNIEXPORT jlong JNICALL -Java_com_example_llama_Llm_new_1context(JNIEnv *env, jobject, jlong jmodel) { +Java_android_llama_cpp_LLamaAndroid_new_1context(JNIEnv *env, jobject, jlong jmodel) { auto model = reinterpret_cast(jmodel); if (!model) { @@ -139,25 +139,25 @@ Java_com_example_llama_Llm_new_1context(JNIEnv *env, jobject, jlong jmodel) { extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_free_1context(JNIEnv *, jobject, jlong context) { +Java_android_llama_cpp_LLamaAndroid_free_1context(JNIEnv *, jobject, jlong context) { llama_free(reinterpret_cast(context)); } extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_backend_1free(JNIEnv *, jobject) { +Java_android_llama_cpp_LLamaAndroid_backend_1free(JNIEnv *, jobject) { llama_backend_free(); } extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_log_1to_1android(JNIEnv *, jobject) { +Java_android_llama_cpp_LLamaAndroid_log_1to_1android(JNIEnv *, jobject) { llama_log_set(log_callback, NULL); } extern "C" JNIEXPORT jstring JNICALL -Java_com_example_llama_Llm_bench_1model( +Java_android_llama_cpp_LLamaAndroid_bench_1model( JNIEnv *env, jobject, jlong context_pointer, @@ -271,13 +271,13 @@ Java_com_example_llama_Llm_bench_1model( extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_free_1batch(JNIEnv *, jobject, jlong batch_pointer) { +Java_android_llama_cpp_LLamaAndroid_free_1batch(JNIEnv *, jobject, jlong batch_pointer) { llama_batch_free(*reinterpret_cast(batch_pointer)); } extern "C" JNIEXPORT jlong JNICALL -Java_com_example_llama_Llm_new_1batch(JNIEnv *, jobject, jint n_tokens, jint embd, jint n_seq_max) { +Java_android_llama_cpp_LLamaAndroid_new_1batch(JNIEnv *, jobject, jint n_tokens, jint embd, jint n_seq_max) { // Source: Copy of llama.cpp:llama_batch_init but heap-allocated. @@ -313,19 +313,19 @@ Java_com_example_llama_Llm_new_1batch(JNIEnv *, jobject, jint n_tokens, jint emb extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_backend_1init(JNIEnv *, jobject) { +Java_android_llama_cpp_LLamaAndroid_backend_1init(JNIEnv *, jobject) { llama_backend_init(); } extern "C" JNIEXPORT jstring JNICALL -Java_com_example_llama_Llm_system_1info(JNIEnv *env, jobject) { +Java_android_llama_cpp_LLamaAndroid_system_1info(JNIEnv *env, jobject) { return env->NewStringUTF(llama_print_system_info()); } extern "C" JNIEXPORT jint JNICALL -Java_com_example_llama_Llm_completion_1init( +Java_android_llama_cpp_LLamaAndroid_completion_1init( JNIEnv *env, jobject, jlong context_pointer, @@ -376,7 +376,7 @@ Java_com_example_llama_Llm_completion_1init( extern "C" JNIEXPORT jstring JNICALL -Java_com_example_llama_Llm_completion_1loop( +Java_android_llama_cpp_LLamaAndroid_completion_1loop( JNIEnv * env, jobject, jlong context_pointer, @@ -438,6 +438,6 @@ Java_com_example_llama_Llm_completion_1loop( extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_kv_1cache_1clear(JNIEnv *, jobject, jlong context) { +Java_android_llama_cpp_LLamaAndroid_kv_1cache_1clear(JNIEnv *, jobject, jlong context) { llama_kv_cache_clear(reinterpret_cast(context)); } diff --git a/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt b/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt similarity index 97% rename from examples/llama.android/app/src/main/java/com/example/llama/Llm.kt rename to examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt index d86afee37..6c63e54e0 100644 --- a/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt +++ b/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt @@ -1,4 +1,4 @@ -package com.example.llama +package android.llama.cpp import android.util.Log import kotlinx.coroutines.CoroutineDispatcher @@ -10,7 +10,7 @@ import kotlinx.coroutines.withContext import java.util.concurrent.Executors import kotlin.concurrent.thread -class Llm { +class LLamaAndroid { private val tag: String? = this::class.simpleName private val threadLocalState: ThreadLocal = ThreadLocal.withInitial { State.Idle } @@ -165,8 +165,8 @@ class Llm { } // Enforce only one instance of Llm. - private val _instance: Llm = Llm() + private val _instance: LLamaAndroid = LLamaAndroid() - fun instance(): Llm = _instance + fun instance(): LLamaAndroid = _instance } } diff --git a/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt b/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt new file mode 100644 index 000000000..cbbb974d3 --- /dev/null +++ b/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt @@ -0,0 +1,17 @@ +package android.llama.cpp + +import org.junit.Test + +import org.junit.Assert.* + +/** + * Example local unit test, which will execute on the development machine (host). + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +class ExampleUnitTest { + @Test + fun addition_isCorrect() { + assertEquals(4, 2 + 2) + } +} diff --git a/examples/llama.android/settings.gradle.kts b/examples/llama.android/settings.gradle.kts index 2ba32c4fa..c7c1a034a 100644 --- a/examples/llama.android/settings.gradle.kts +++ b/examples/llama.android/settings.gradle.kts @@ -15,3 +15,4 @@ dependencyResolutionManagement { rootProject.name = "LlamaAndroid" include(":app") +include(":llama") From 541600201e6480f54ae09e58d16b154d4b4b331d Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 14 May 2024 09:33:42 +0200 Subject: [PATCH 015/137] llama : disable pipeline parallelism with nkvo (#7265) --- llama.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 01a35dfb6..ad35e4a2e 100644 --- a/llama.cpp +++ b/llama.cpp @@ -15849,7 +15849,11 @@ struct llama_context * llama_new_context_with_model( ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead_custom(LLAMA_MAX_NODES, false)); // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary - bool pipeline_parallel = llama_get_device_count() > 1 && model->n_gpu_layers > (int)model->hparams.n_layer && model->split_mode == LLAMA_SPLIT_MODE_LAYER; + bool pipeline_parallel = + llama_get_device_count() > 1 && + model->n_gpu_layers > (int)model->hparams.n_layer && + model->split_mode == LLAMA_SPLIT_MODE_LAYER && + params.offload_kqv; #ifndef GGML_USE_CUDA // pipeline parallelism requires support for async compute and events // currently this is only implemented in the CUDA backend From 5e31828d3e35c76ecfee665bc23771a4bec1d130 Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Tue, 14 May 2024 14:27:19 +0300 Subject: [PATCH 016/137] ggml : add RPC backend (#6829) * ggml : add RPC backend The RPC backend proxies all operations to a remote server which runs a regular backend (CPU, CUDA, Metal, etc). * set TCP_NODELAY * add CI workflows * Address review comments * fix warning * implement llama_max_devices() for RPC * Address review comments * Address review comments * wrap sockfd into a struct * implement get_alignment and get_max_size * add get_device_memory * fix warning * win32 support * add README * readme : trim trailing whitespace * Address review comments * win32 fix * Address review comments * fix compile warnings on macos --- .github/workflows/build.yml | 32 ++ CMakeLists.txt | 13 + common/common.cpp | 10 + common/common.h | 1 + examples/CMakeLists.txt | 3 + examples/rpc/CMakeLists.txt | 2 + examples/rpc/README.md | 74 +++ examples/rpc/rpc-server.cpp | 70 +++ ggml-rpc.cpp | 1023 +++++++++++++++++++++++++++++++++++ ggml-rpc.h | 24 + llama.cpp | 238 ++++---- llama.h | 3 + 12 files changed, 1395 insertions(+), 98 deletions(-) create mode 100644 examples/rpc/CMakeLists.txt create mode 100644 examples/rpc/README.md create mode 100644 examples/rpc/rpc-server.cpp create mode 100644 ggml-rpc.cpp create mode 100644 ggml-rpc.h diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8c338534d..7ac0e5f6e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -340,6 +340,36 @@ jobs: cd build ctest -L main --verbose + ubuntu-latest-cmake-rpc: + runs-on: ubuntu-latest + + continue-on-error: true + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential + + - name: Build + id: cmake_build + run: | + mkdir build + cd build + cmake -DLLAMA_RPC=ON .. + cmake --build . --config Release -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose + ubuntu-22-cmake-vulkan: runs-on: ubuntu-22.04 @@ -663,6 +693,8 @@ jobs: strategy: matrix: include: + - build: 'rpc' + defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_RPC=ON -DBUILD_SHARED_LIBS=ON' - build: 'noavx' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DBUILD_SHARED_LIBS=ON' - build: 'avx2' diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c3b5c8e4..feb6f39d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,7 @@ set(LLAMA_METAL_MACOSX_VERSION_MIN "" CACHE STRING set(LLAMA_METAL_STD "" CACHE STRING "llama: metal standard version (-std flag)") option(LLAMA_KOMPUTE "llama: use Kompute" OFF) option(LLAMA_MPI "llama: use MPI" OFF) +option(LLAMA_RPC "llama: use RPC" OFF) option(LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF) option(LLAMA_SYCL "llama: use SYCL" OFF) option(LLAMA_SYCL_F16 "llama: use 16 bit floats for sycl calculations" OFF) @@ -494,6 +495,17 @@ if (LLAMA_MPI) endif() endif() +if (LLAMA_RPC) + add_compile_definitions(GGML_USE_RPC) + + if (WIN32) + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ws2_32) + endif() + + set(GGML_HEADERS_RPC ggml-rpc.h) + set(GGML_SOURCES_RPC ggml-rpc.cpp) +endif() + if (LLAMA_CLBLAST) find_package(CLBlast) if (CLBlast_FOUND) @@ -1176,6 +1188,7 @@ add_library(ggml OBJECT ${GGML_SOURCES_OPENCL} ${GGML_HEADERS_OPENCL} ${GGML_SOURCES_METAL} ${GGML_HEADERS_METAL} ${GGML_SOURCES_MPI} ${GGML_HEADERS_MPI} + ${GGML_SOURCES_RPC} ${GGML_HEADERS_RPC} ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA} ${GGML_SOURCES_SYCL} ${GGML_HEADERS_SYCL} ${GGML_SOURCES_KOMPUTE} ${GGML_HEADERS_KOMPUTE} diff --git a/common/common.cpp b/common/common.cpp index ba1ecf0e5..96130ad54 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1060,6 +1060,14 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa #endif // GGML_USE_CUDA_SYCL_VULKAN return true; } + if (arg == "--rpc") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.rpc_servers = argv[i]; + return true; + } if (arg == "--no-mmap") { params.use_mmap = false; return true; @@ -1557,6 +1565,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n"); printf(" or for intermediate results and KV (with split-mode = row) (default: %d)\n", params.main_gpu); } + printf(" --rpc SERVERS comma separated list of RPC servers\n"); printf(" --verbose-prompt print a verbose prompt before generation (default: %s)\n", params.verbose_prompt ? "true" : "false"); printf(" --no-display-prompt don't print prompt at generation (default: %s)\n", !params.display_prompt ? "true" : "false"); printf(" -gan N, --grp-attn-n N\n"); @@ -1830,6 +1839,7 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params & if (params.n_gpu_layers != -1) { mparams.n_gpu_layers = params.n_gpu_layers; } + mparams.rpc_servers = params.rpc_servers.c_str(); mparams.main_gpu = params.main_gpu; mparams.split_mode = params.split_mode; mparams.tensor_split = params.tensor_split; diff --git a/common/common.h b/common/common.h index d80344f2a..566490e2f 100644 --- a/common/common.h +++ b/common/common.h @@ -82,6 +82,7 @@ struct gpt_params { float yarn_beta_slow = 1.0f; // YaRN high correction dim int32_t yarn_orig_ctx = 0; // YaRN original context length float defrag_thold = -1.0f; // KV cache defragmentation threshold + std::string rpc_servers = ""; // comma separated list of RPC servers ggml_backend_sched_eval_callback cb_eval = nullptr; void * cb_eval_user_data = nullptr; diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index f421769cc..b40ee4ccb 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -49,4 +49,7 @@ else() add_subdirectory(server) endif() add_subdirectory(export-lora) + if (LLAMA_RPC) + add_subdirectory(rpc) + endif() endif() diff --git a/examples/rpc/CMakeLists.txt b/examples/rpc/CMakeLists.txt new file mode 100644 index 000000000..ae48fb98d --- /dev/null +++ b/examples/rpc/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(rpc-server rpc-server.cpp) +target_link_libraries(rpc-server PRIVATE ggml llama) diff --git a/examples/rpc/README.md b/examples/rpc/README.md new file mode 100644 index 000000000..325d0abc4 --- /dev/null +++ b/examples/rpc/README.md @@ -0,0 +1,74 @@ +## Overview + +The `rpc-server` allows running `ggml` backend on a remote host. +The RPC backend communicates with one or several instances of `rpc-server` and offloads computations to them. +This can be used for distributed LLM inference with `llama.cpp` in the following way: + +```mermaid +flowchart TD + rpcb---|TCP|srva + rpcb---|TCP|srvb + rpcb-.-|TCP|srvn + subgraph hostn[Host N] + srvn[rpc-server]-.-backend3["Backend (CUDA,Metal,etc.)"] + end + subgraph hostb[Host B] + srvb[rpc-server]---backend2["Backend (CUDA,Metal,etc.)"] + end + subgraph hosta[Host A] + srva[rpc-server]---backend["Backend (CUDA,Metal,etc.)"] + end + subgraph host[Main Host] + ggml[llama.cpp]---rpcb[RPC backend] + end + style hostn stroke:#66,stroke-width:2px,stroke-dasharray: 5 5 +``` + +Each host can run a different backend, e.g. one with CUDA and another with Metal. +You can also run multiple `rpc-server` instances on the same host, each with a different backend. + +## Usage + +On each host, build the corresponding backend with `cmake` and add `-DLLAMA_RPC=ON` to the build options. +For example, to build the CUDA backend with RPC support: + +```bash +mkdir build-rpc-cuda +cd build-rpc-cuda +cmake .. -DLLAMA_CUDA=ON -DLLAMA_RPC=ON +cmake --build . --config Release +``` + +Then, start the `rpc-server` with the backend: + +```bash +$ bin/rpc-server 0.0.0.0 50052 +create_backend: using CUDA backend +ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no +ggml_cuda_init: CUDA_USE_TENSOR_CORES: yes +ggml_cuda_init: found 1 CUDA devices: + Device 0: NVIDIA T1200 Laptop GPU, compute capability 7.5, VMM: yes +Starting RPC server on 0.0.0.0:50052 +``` + +When using the CUDA backend, you can specify the device with the `CUDA_VISIBLE_DEVICES` environment variable, e.g.: +```bash +$ CUDA_VISIBLE_DEVICES=0 bin/rpc-server 0.0.0.0 50052 +``` +This way you can run multiple `rpc-server` instances on the same host, each with a different CUDA device. + + +On the main host build `llama.cpp` only with `-DLLAMA_RPC=ON`: + +```bash +mkdir build-rpc +cd build-rpc +cmake .. -DLLAMA_RPC=ON +cmake --build . --config Release +``` + +Finally, use the `--rpc` option to specify the host and port of each `rpc-server`: + +```bash +$ bin/main -m ../models/tinyllama-1b/ggml-model-f16.gguf -p "Hello, my name is" --repeat-penalty 1.0 -n 64 --rpc 192.168.88.10:50052,192.168.88.11:50052 -ngl 99 +``` diff --git a/examples/rpc/rpc-server.cpp b/examples/rpc/rpc-server.cpp new file mode 100644 index 000000000..496af8496 --- /dev/null +++ b/examples/rpc/rpc-server.cpp @@ -0,0 +1,70 @@ +#ifdef GGML_USE_CUDA +#include "ggml-cuda.h" +#endif + +#ifdef GGML_USE_METAL +#include "ggml-metal.h" +#endif + +#include "ggml-rpc.h" +#include +#include + +static ggml_backend_t create_backend() { + ggml_backend_t backend = NULL; +#ifdef GGML_USE_CUDA + fprintf(stderr, "%s: using CUDA backend\n", __func__); + backend = ggml_backend_cuda_init(0); // init device 0 + if (!backend) { + fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__); + } +#elif GGML_USE_METAL + fprintf(stderr, "%s: using Metal backend\n", __func__); + backend = ggml_backend_metal_init(); + if (!backend) { + fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__); + } +#endif + + // if there aren't GPU Backends fallback to CPU backend + if (!backend) { + fprintf(stderr, "%s: using CPU backend\n", __func__); + backend = ggml_backend_cpu_init(); + } + return backend; +} + +static void get_backend_memory(size_t * free_mem, size_t * total_mem) { +#ifdef GGML_USE_CUDA + ggml_backend_cuda_get_device_memory(0, free_mem, total_mem); +#else + // TODO: implement for other backends + *free_mem = 1; + *total_mem = 1; +#endif +} + +int main(int argc, char * argv[]) { + if (argc < 3) { + fprintf(stderr, "Usage: %s \n", argv[0]); + return 1; + } + const char * host = argv[1]; + int port = std::stoi(argv[2]); + if (port <= 0 || port > 65535) { + fprintf(stderr, "Invalid port number: %d\n", port); + return 1; + } + ggml_backend_t backend = create_backend(); + if (!backend) { + fprintf(stderr, "Failed to create backend\n"); + return 1; + } + printf("Starting RPC server on %s:%d\n", host, port); + size_t free_mem, total_mem; + get_backend_memory(&free_mem, &total_mem); + std::string endpoint = std::string(host) + ":" + std::to_string(port); + start_rpc_server(backend, endpoint.c_str(), free_mem, total_mem); + ggml_backend_free(backend); + return 0; +} diff --git a/ggml-rpc.cpp b/ggml-rpc.cpp new file mode 100644 index 000000000..efeacb297 --- /dev/null +++ b/ggml-rpc.cpp @@ -0,0 +1,1023 @@ +#include "ggml-rpc.h" +#include "ggml.h" +#include "ggml-backend-impl.h" + +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include +# include +#else +# include +# include +# include +# include +# include +# include +# include +#endif +#include + +#define UNUSED GGML_UNUSED + +#define GGML_DEBUG 1 +#if (GGML_DEBUG >= 1) +#define GGML_PRINT_DEBUG(...) printf(__VA_ARGS__) +#else +#define GGML_PRINT_DEBUG(...) +#endif + +#ifdef _WIN32 +typedef SOCKET sockfd_t; +using ssize_t = __int64; +#else +typedef int sockfd_t; +#endif + +// cross-platform socket +struct socket_t { + sockfd_t fd; + socket_t(sockfd_t fd) : fd(fd) {} + ~socket_t() { +#ifdef _WIN32 + closesocket(this->fd); +#else + close(this->fd); +#endif + } +}; + +// ggml_tensor is serialized into rpc_tensor +struct rpc_tensor { + uint64_t id; + uint32_t type; + uint64_t buffer; + uint32_t ne[GGML_MAX_DIMS]; + uint32_t nb[GGML_MAX_DIMS]; + uint32_t op; + int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; + int32_t flags; + uint64_t src[GGML_MAX_SRC]; + uint64_t view_src; + uint64_t view_offs; + uint64_t data; + char name[GGML_MAX_NAME]; +}; + +// RPC commands +enum rpc_cmd { + ALLOC_BUFFER = 0, + GET_ALIGNMENT, + GET_MAX_SIZE, + BUFFER_GET_BASE, + FREE_BUFFER, + BUFFER_CLEAR, + SET_TENSOR, + GET_TENSOR, + COPY_TENSOR, + GRAPH_COMPUTE, + GET_DEVICE_MEMORY, +}; + +// RPC data structures + +static ggml_guid_t ggml_backend_rpc_guid() { + static ggml_guid guid = {0x99, 0x68, 0x5b, 0x6c, 0xd2, 0x83, 0x3d, 0x24, 0x25, 0x36, 0x72, 0xe1, 0x5b, 0x0e, 0x14, 0x03}; + return &guid; +} + +struct ggml_backend_rpc_buffer_type_context { + std::shared_ptr sock; + std::string name; + size_t alignment; + size_t max_size; +}; + +struct ggml_backend_rpc_context { + std::string endpoint; + std::string name; + std::shared_ptr sock; + ggml_backend_buffer_type_t buft; +}; + +struct ggml_backend_rpc_buffer_context { + std::shared_ptr sock; + std::unordered_map base_cache; + uint64_t remote_ptr; + std::string name; +}; + +// RPC helper functions + +static std::shared_ptr make_socket(sockfd_t fd) { +#ifdef _WIN32 + if (fd == INVALID_SOCKET) { + return nullptr; + } +#else + if (fd < 0) { + return nullptr; + } +#endif + return std::make_shared(fd); +} + +static bool set_no_delay(sockfd_t sockfd) { + int flag = 1; + // set TCP_NODELAY to disable Nagle's algorithm + int ret = setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(int)); + return ret >= 0; +} + +static std::shared_ptr socket_connect(const char * host, int port) { + struct sockaddr_in addr; + auto sockfd = socket(AF_INET, SOCK_STREAM, 0); + auto sock_ptr = make_socket(sockfd); + if (sock_ptr == nullptr) { + return nullptr; + } + if (!set_no_delay(sockfd)) { + fprintf(stderr, "Failed to set TCP_NODELAY\n"); + return nullptr; + } + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + struct hostent * server = gethostbyname(host); + if (server == NULL) { + fprintf(stderr, "Cannot resolve host '%s'\n", host); + return nullptr; + } + memcpy(&addr.sin_addr.s_addr, server->h_addr, server->h_length); + if (connect(sock_ptr->fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + return nullptr; + } + return sock_ptr; +} + +static std::shared_ptr socket_accept(sockfd_t srv_sockfd) { + auto client_socket_fd = accept(srv_sockfd, NULL, NULL); + auto client_socket = make_socket(client_socket_fd); + if (client_socket == nullptr) { + return nullptr; + } + if (!set_no_delay(client_socket_fd)) { + fprintf(stderr, "Failed to set TCP_NODELAY\n"); + return nullptr; + } + return client_socket; +} + +static std::shared_ptr create_server_socket(const char * host, int port) { + auto sockfd = socket(AF_INET, SOCK_STREAM, 0); + auto sock = make_socket(sockfd); + if (sock == nullptr) { + return nullptr; + } + + struct sockaddr_in serv_addr; + serv_addr.sin_family = AF_INET; + serv_addr.sin_addr.s_addr = inet_addr(host); + serv_addr.sin_port = htons(port); + + if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + return nullptr; + } + if (listen(sockfd, 1) < 0) { + return nullptr; + } + return sock; +} + +static bool send_data(sockfd_t sockfd, const void * data, size_t size) { + size_t bytes_sent = 0; + while (bytes_sent < size) { + ssize_t n = send(sockfd, (const char *)data + bytes_sent, size - bytes_sent, 0); + if (n < 0) { + return false; + } + bytes_sent += n; + } + return true; +} + +static bool recv_data(sockfd_t sockfd, void * data, size_t size) { + size_t bytes_recv = 0; + while (bytes_recv < size) { + ssize_t n = recv(sockfd, (char *)data + bytes_recv, size - bytes_recv, 0); + if (n <= 0) { + return false; + } + bytes_recv += n; + } + return true; +} + +static bool parse_endpoint(const char * endpoint, std::string & host, int & port) { + std::string str(endpoint); + size_t pos = str.find(':'); + if (pos == std::string::npos) { + return false; + } + host = str.substr(0, pos); + port = std::stoi(str.substr(pos + 1)); + return true; +} + +// RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) | +// RPC response: | response_size (8 bytes) | response_data (response_size bytes) | +static bool send_rpc_cmd(const std::shared_ptr & sock, enum rpc_cmd cmd, const std::vector & input, std::vector & output) { + uint8_t cmd_byte = cmd; + if (!send_data(sock->fd, &cmd_byte, sizeof(cmd_byte))) { + return false; + } + uint64_t input_size = input.size(); + if (!send_data(sock->fd, &input_size, sizeof(input_size))) { + return false; + } + if (!send_data(sock->fd, input.data(), input.size())) { + return false; + } + uint64_t output_size; + if (!recv_data(sock->fd, &output_size, sizeof(output_size))) { + return false; + } + if (output_size == 0) { + output.clear(); + return true; + } + output.resize(output_size); + if (!recv_data(sock->fd, output.data(), output_size)) { + return false; + } + return true; +} + +// RPC client-side implementation + +GGML_CALL static const char * ggml_backend_rpc_buffer_get_name(ggml_backend_buffer_t buffer) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + return ctx->name.c_str(); +} + +GGML_CALL static void ggml_backend_rpc_buffer_free_buffer(ggml_backend_buffer_t buffer) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + // input serialization format: | remote_ptr (8 bytes) | + std::vector input(sizeof(uint64_t), 0); + uint64_t remote_ptr = ctx->remote_ptr; + memcpy(input.data(), &remote_ptr, sizeof(remote_ptr)); + std::vector output; + bool status = send_rpc_cmd(ctx->sock, FREE_BUFFER, input, output); + GGML_ASSERT(status); + GGML_ASSERT(output.empty()); + delete ctx; +} + +GGML_CALL static void * ggml_backend_rpc_buffer_get_base(ggml_backend_buffer_t buffer) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + if (ctx->base_cache.find(buffer) != ctx->base_cache.end()) { + return ctx->base_cache[buffer]; + } + // input serialization format: | remote_ptr (8 bytes) | + std::vector input(sizeof(uint64_t), 0); + uint64_t remote_ptr = ctx->remote_ptr; + memcpy(input.data(), &remote_ptr, sizeof(remote_ptr)); + std::vector output; + bool status = send_rpc_cmd(ctx->sock, BUFFER_GET_BASE, input, output); + GGML_ASSERT(status); + GGML_ASSERT(output.size() == sizeof(uint64_t)); + // output serialization format: | base_ptr (8 bytes) | + uint64_t base_ptr; + memcpy(&base_ptr, output.data(), sizeof(base_ptr)); + void * base = reinterpret_cast(base_ptr); + ctx->base_cache[buffer] = base; + return base; +} + +static rpc_tensor serialize_tensor(const ggml_tensor * tensor) { + rpc_tensor result; + result.id = reinterpret_cast(tensor); + result.type = tensor->type; + if (tensor->buffer) { + ggml_backend_buffer_t buffer = tensor->buffer; + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + result.buffer = ctx->remote_ptr; + } else { + result.buffer = 0; + } + for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) { + result.ne[i] = tensor->ne[i]; + result.nb[i] = tensor->nb[i]; + } + result.op = tensor->op; + for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) { + result.op_params[i] = tensor->op_params[i]; + } + result.flags = tensor->flags; + for (uint32_t i = 0; i < GGML_MAX_SRC; i++) { + result.src[i] = reinterpret_cast(tensor->src[i]); + } + result.view_src = reinterpret_cast(tensor->view_src); + result.view_offs = tensor->view_offs; + result.data = reinterpret_cast(tensor->data); + snprintf(result.name, GGML_MAX_NAME, "%s", tensor->name); + return result; +} + +static ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) { + ggml_tensor * result = ggml_new_tensor_4d(ctx, (ggml_type) tensor->type, + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); + for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) { + result->nb[i] = tensor->nb[i]; + } + result->buffer = reinterpret_cast(tensor->buffer); + result->op = (ggml_op) tensor->op; + for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) { + result->op_params[i] = tensor->op_params[i]; + } + result->flags = tensor->flags; + result->data = reinterpret_cast(tensor->data); + ggml_set_name(result, tensor->name); + return result; +} + +GGML_CALL static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { + UNUSED(buffer); + if (ggml_is_quantized(tensor->type)) { + // TODO: this check is due to MATRIX_ROW_PADDING in CUDA and should be generalized + GGML_ASSERT(tensor->ne[0] % 512 == 0 && "unsupported quantized tensor"); + } +} + +GGML_CALL static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + // input serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) | + size_t input_size = sizeof(rpc_tensor) + sizeof(uint64_t) + size; + std::vector input(input_size, 0); + rpc_tensor rpc_tensor = serialize_tensor(tensor); + memcpy(input.data(), &rpc_tensor, sizeof(rpc_tensor)); + memcpy(input.data() + sizeof(rpc_tensor), &offset, sizeof(offset)); + memcpy(input.data() + sizeof(rpc_tensor) + sizeof(offset), data, size); + std::vector output; + bool status = send_rpc_cmd(ctx->sock, SET_TENSOR, input, output); + GGML_ASSERT(status); +} + +GGML_CALL static void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + // input serialization format: | rpc_tensor | offset (8 bytes) | size (8 bytes) | + int input_size = sizeof(rpc_tensor) + 2*sizeof(uint64_t); + std::vector input(input_size, 0); + rpc_tensor rpc_tensor = serialize_tensor(tensor); + memcpy(input.data(), &rpc_tensor, sizeof(rpc_tensor)); + memcpy(input.data() + sizeof(rpc_tensor), &offset, sizeof(offset)); + memcpy(input.data() + sizeof(rpc_tensor) + sizeof(offset), &size, sizeof(size)); + std::vector output; + bool status = send_rpc_cmd(ctx->sock, GET_TENSOR, input, output); + GGML_ASSERT(status); + GGML_ASSERT(output.size() == size); + // output serialization format: | data (size bytes) | + memcpy(data, output.data(), size); +} + +GGML_CALL static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) { + // check if src and dst are on the same server + ggml_backend_buffer_t src_buffer = src->buffer; + ggml_backend_rpc_buffer_context * src_ctx = (ggml_backend_rpc_buffer_context *)src_buffer->context; + ggml_backend_buffer_t dst_buffer = dst->buffer; + ggml_backend_rpc_buffer_context * dst_ctx = (ggml_backend_rpc_buffer_context *)dst_buffer->context; + if (src_ctx->sock != dst_ctx->sock) { + return false; + } + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + // input serialization format: | rpc_tensor src | rpc_tensor dst | + int input_size = 2*sizeof(rpc_tensor); + std::vector input(input_size, 0); + rpc_tensor rpc_src = serialize_tensor(src); + rpc_tensor rpc_dst = serialize_tensor(dst); + memcpy(input.data(), &rpc_src, sizeof(rpc_src)); + memcpy(input.data() + sizeof(rpc_src), &rpc_dst, sizeof(rpc_dst)); + std::vector output; + bool status = send_rpc_cmd(ctx->sock, COPY_TENSOR, input, output); + GGML_ASSERT(status); + // output serialization format: | result (1 byte) | + GGML_ASSERT(output.size() == 1); + return output[0]; +} + +GGML_CALL static void ggml_backend_rpc_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + // serialization format: | bufptr (8 bytes) | value (1 byte) | + int input_size = sizeof(uint64_t) + sizeof(uint8_t); + std::vector input(input_size, 0); + memcpy(input.data(), &ctx->remote_ptr, sizeof(ctx->remote_ptr)); + memcpy(input.data() + sizeof(ctx->remote_ptr), &value, sizeof(value)); + std::vector output; + bool status = send_rpc_cmd(ctx->sock, BUFFER_CLEAR, input, output); + GGML_ASSERT(status); +} + +static ggml_backend_buffer_i ggml_backend_rpc_buffer_interface = { + /* .get_name = */ ggml_backend_rpc_buffer_get_name, + /* .free_buffer = */ ggml_backend_rpc_buffer_free_buffer, + /* .get_base = */ ggml_backend_rpc_buffer_get_base, + /* .init_tensor = */ ggml_backend_rpc_buffer_init_tensor, + /* .set_tensor = */ ggml_backend_rpc_buffer_set_tensor, + /* .get_tensor = */ ggml_backend_rpc_buffer_get_tensor, + /* .cpy_tensor = */ ggml_backend_rpc_buffer_cpy_tensor, + /* .clear = */ ggml_backend_rpc_buffer_clear, + /* .reset = */ NULL, +}; + +GGML_CALL static const char * ggml_backend_rpc_buffer_type_name(ggml_backend_buffer_type_t buft) { + ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; + return buft_ctx->name.c_str(); +} + +GGML_CALL static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) { + ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; + // input serialization format: | size (8 bytes) | + int input_size = sizeof(uint64_t); + std::vector input(input_size, 0); + memcpy(input.data(), &size, sizeof(size)); + std::vector output; + bool status = send_rpc_cmd(buft_ctx->sock, ALLOC_BUFFER, input, output); + GGML_ASSERT(status); + GGML_ASSERT(output.size() == 2*sizeof(uint64_t)); + // output serialization format: | remote_ptr (8 bytes) | remote_size (8 bytes) | + uint64_t remote_ptr; + memcpy(&remote_ptr, output.data(), sizeof(remote_ptr)); + size_t remote_size; + memcpy(&remote_size, output.data() + sizeof(uint64_t), sizeof(remote_size)); + + ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft, + ggml_backend_rpc_buffer_interface, + new ggml_backend_rpc_buffer_context{buft_ctx->sock, {}, remote_ptr, "RPC"}, + remote_size); + + return buffer; +} + +static size_t get_alignment(const std::shared_ptr & sock) { + // input serialization format: | 0 bytes | + std::vector input; + std::vector output; + bool status = send_rpc_cmd(sock, GET_ALIGNMENT, input, output); + GGML_ASSERT(status); + GGML_ASSERT(output.size() == sizeof(uint64_t)); + // output serialization format: | alignment (8 bytes) | + uint64_t alignment; + memcpy(&alignment, output.data(), sizeof(alignment)); + return alignment; +} + +GGML_CALL static size_t ggml_backend_rpc_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) { + ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; + return buft_ctx->alignment; +} + +static size_t get_max_size(const std::shared_ptr & sock) { + // input serialization format: | 0 bytes | + std::vector input; + std::vector output; + bool status = send_rpc_cmd(sock, GET_MAX_SIZE, input, output); + GGML_ASSERT(status); + GGML_ASSERT(output.size() == sizeof(uint64_t)); + // output serialization format: | max_size (8 bytes) | + uint64_t max_size; + memcpy(&max_size, output.data(), sizeof(max_size)); + return max_size; +} + +GGML_CALL static size_t ggml_backend_rpc_get_max_size(ggml_backend_buffer_type_t buft) { + ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; + return buft_ctx->max_size; +} + +GGML_CALL static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) { + UNUSED(buft); + return ggml_nbytes(tensor); +} + +GGML_CALL static bool ggml_backend_rpc_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) { + if (!ggml_backend_is_rpc(backend)) { + return false; + } + ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context; + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; + return buft_ctx->sock == rpc_ctx->sock; +} + +static ggml_backend_buffer_type_i ggml_backend_rpc_buffer_type_interface = { + /* .get_name = */ ggml_backend_rpc_buffer_type_name, + /* .alloc_buffer = */ ggml_backend_rpc_buffer_type_alloc_buffer, + /* .get_alignment = */ ggml_backend_rpc_buffer_type_get_alignment, + /* .get_max_size = */ ggml_backend_rpc_get_max_size, + /* .get_alloc_size = */ ggml_backend_rpc_buffer_type_get_alloc_size, + /* .supports_backend = */ ggml_backend_rpc_buffer_type_supports_backend, + /* .is_host = */ NULL, +}; + + +GGML_CALL static const char * ggml_backend_rpc_name(ggml_backend_t backend) { + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; + + return rpc_ctx->name.c_str(); +} + +GGML_CALL static void ggml_backend_rpc_free(ggml_backend_t backend) { + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; + ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)rpc_ctx->buft->context; + delete buft_ctx; + delete rpc_ctx->buft; + delete rpc_ctx; + delete backend; +} + +GGML_CALL static ggml_backend_buffer_type_t ggml_backend_rpc_get_default_buffer_type(ggml_backend_t backend) { + ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context; + return ctx->buft; +} + +GGML_CALL static void ggml_backend_rpc_synchronize(ggml_backend_t backend) { + UNUSED(backend); + // this is no-op because we don't have any async operations +} + +static void add_tensor(ggml_tensor * tensor, std::vector & tensors, std::unordered_set & visited) { + if (tensor == nullptr) { + return; + } + if (visited.find(tensor) != visited.end()) { + return; + } + visited.insert(tensor); + for (int i = 0; i < GGML_MAX_SRC; i++) { + add_tensor(tensor->src[i], tensors, visited); + } + add_tensor(tensor->view_src, tensors, visited); + tensors.push_back(serialize_tensor(tensor)); +} + +static void serialize_graph(const ggml_cgraph * cgraph, std::vector & output) { + uint32_t n_nodes = cgraph->n_nodes; + std::vector tensors; + std::unordered_set visited; + for (uint32_t i = 0; i < n_nodes; i++) { + add_tensor(cgraph->nodes[i], tensors, visited); + } + // serialization format: + // | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | + uint32_t n_tensors = tensors.size(); + int output_size = sizeof(uint32_t) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t) + n_tensors * sizeof(rpc_tensor); + output.resize(output_size, 0); + memcpy(output.data(), &n_nodes, sizeof(n_nodes)); + uint64_t * out_nodes = (uint64_t *)(output.data() + sizeof(n_nodes)); + for (uint32_t i = 0; i < n_nodes; i++) { + out_nodes[i] = reinterpret_cast(cgraph->nodes[i]); + } + uint32_t * out_ntensors = (uint32_t *)(output.data() + sizeof(n_nodes) + n_nodes * sizeof(uint64_t)); + *out_ntensors = n_tensors; + rpc_tensor * out_tensors = (rpc_tensor *)(output.data() + sizeof(n_nodes) + n_nodes * sizeof(uint64_t) + sizeof(uint32_t)); + memcpy(out_tensors, tensors.data(), n_tensors * sizeof(rpc_tensor)); +} + +GGML_CALL static enum ggml_status ggml_backend_rpc_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { + ggml_backend_rpc_context * rpc_ctx = (ggml_backend_rpc_context *)backend->context; + std::vector input; + serialize_graph(cgraph, input); + std::vector output; + bool status = send_rpc_cmd(rpc_ctx->sock, GRAPH_COMPUTE, input, output); + GGML_ASSERT(status); + GGML_ASSERT(output.size() == 1); + return (enum ggml_status)output[0]; +} + +GGML_CALL static bool ggml_backend_rpc_supports_op(ggml_backend_t backend, const ggml_tensor * op) { + UNUSED(backend); + UNUSED(op); + GGML_ASSERT(false && "not implemented"); + return false; +} + +static ggml_backend_i ggml_backend_rpc_interface = { + /* .get_name = */ ggml_backend_rpc_name, + /* .free = */ ggml_backend_rpc_free, + /* .get_default_buffer_type = */ ggml_backend_rpc_get_default_buffer_type, + /* .set_tensor_async = */ NULL, + /* .get_tensor_async = */ NULL, + /* .cpy_tensor_async = */ NULL, + /* .synchronize = */ ggml_backend_rpc_synchronize, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_compute = */ NULL, + /* .graph_compute = */ ggml_backend_rpc_graph_compute, + /* .supports_op = */ ggml_backend_rpc_supports_op, + /* .offload_op = */ NULL, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .event_synchronize = */ NULL, +}; + +static std::unordered_map instances; + +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint) { + ggml_backend_t backend = ggml_backend_rpc_init(endpoint); + return backend != nullptr ? ggml_backend_rpc_get_default_buffer_type(backend) : nullptr; +} + +GGML_CALL ggml_backend_t ggml_backend_rpc_init(const char * endpoint) { + std::string endpoint_str(endpoint); + if (instances.find(endpoint_str) != instances.end()) { + return instances[endpoint_str]; + } +#ifdef _WIN32 + { + WSADATA wsaData; + int res = WSAStartup(MAKEWORD(2, 2), &wsaData); + if (res != 0) { + return nullptr; + } + } +#endif + GGML_PRINT_DEBUG("Connecting to %s\n", endpoint); + std::string host; + int port; + if (!parse_endpoint(endpoint, host, port)) { + return nullptr; + } + auto sock = socket_connect(host.c_str(), port); + if (sock == nullptr) { + return nullptr; + } + size_t alignment = get_alignment(sock); + size_t max_size = get_max_size(sock); + ggml_backend_rpc_buffer_type_context * buft_ctx = new ggml_backend_rpc_buffer_type_context { + /* .sock = */ sock, + /* .name = */ "RPC" + std::to_string(sock->fd), + /* .alignment = */ alignment, + /* .max_size = */ max_size + }; + + ggml_backend_buffer_type_t buft = new ggml_backend_buffer_type { + /* .iface = */ ggml_backend_rpc_buffer_type_interface, + /* .context = */ buft_ctx + }; + + ggml_backend_rpc_context * ctx = new ggml_backend_rpc_context { + /* .endpoint = */ endpoint, + /* .name = */ "RPC" + std::to_string(sock->fd), + /* .sock = */ sock, + /* .buft = */ buft + }; + + instances[endpoint] = new ggml_backend { + /* .guid = */ ggml_backend_rpc_guid(), + /* .interface = */ ggml_backend_rpc_interface, + /* .context = */ ctx + }; + + return instances[endpoint]; +} + +GGML_API GGML_CALL bool ggml_backend_is_rpc(ggml_backend_t backend) { + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_rpc_guid()); +} + +static void get_device_memory(const std::shared_ptr & sock, size_t * free, size_t * total) { + // input serialization format: | 0 bytes | + std::vector input; + std::vector output; + bool status = send_rpc_cmd(sock, GET_DEVICE_MEMORY, input, output); + GGML_ASSERT(status); + GGML_ASSERT(output.size() == 2*sizeof(uint64_t)); + // output serialization format: | free (8 bytes) | total (8 bytes) | + uint64_t free_mem; + memcpy(&free_mem, output.data(), sizeof(free_mem)); + uint64_t total_mem; + memcpy(&total_mem, output.data() + sizeof(uint64_t), sizeof(total_mem)); + *free = free_mem; + *total = total_mem; +} + +GGML_API GGML_CALL void ggml_backend_rpc_get_device_memory(const char * endpoint, size_t * free, size_t * total) { + ggml_backend_t backend = ggml_backend_rpc_init(endpoint); + if (backend == nullptr) { + *free = 0; + *total = 0; + return; + } + ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context; + get_device_memory(ctx->sock, free, total); +} + +// RPC server-side implementation + +static void rpc_alloc_buffer(ggml_backend_t backend, const std::vector & input, std::vector & output) { + // input serialization format: | size (8 bytes) | + uint64_t size; + memcpy(&size, input.data(), sizeof(size)); + ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size); + uint64_t remote_ptr = reinterpret_cast(buffer); + uint64_t remote_size = buffer->size; + GGML_PRINT_DEBUG("[%s] size: %" PRIu64 " -> remote_ptr: %" PRIx64 ", remote_size: %" PRIu64 "\n", __func__, size, remote_ptr, remote_size); + // output serialization format: | remote_ptr (8 bytes) | remote_size (8 bytes) | + output.resize(2*sizeof(uint64_t), 0); + memcpy(output.data(), &remote_ptr, sizeof(remote_ptr)); + memcpy(output.data() + sizeof(uint64_t), &remote_size, sizeof(remote_size)); +} + +static void rpc_get_alignment(ggml_backend_t backend, std::vector & output) { + ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); + size_t alignment = ggml_backend_buft_get_alignment(buft); + GGML_PRINT_DEBUG("[%s] alignment: %lu\n", __func__, alignment); + // output serialization format: | alignment (8 bytes) | + output.resize(sizeof(uint64_t), 0); + memcpy(output.data(), &alignment, sizeof(alignment)); +} + +static void rpc_get_max_size(ggml_backend_t backend, std::vector & output) { + ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); + size_t max_size = ggml_backend_buft_get_max_size(buft); + GGML_PRINT_DEBUG("[%s] max_size: %lu\n", __func__, max_size); + // output serialization format: | max_size (8 bytes) | + output.resize(sizeof(uint64_t), 0); + memcpy(output.data(), &max_size, sizeof(max_size)); +} + +static void rpc_buffer_get_base(const std::vector & input, std::vector & output) { + // input serialization format: | remote_ptr (8 bytes) | + uint64_t remote_ptr; + memcpy(&remote_ptr, input.data(), sizeof(remote_ptr)); + GGML_PRINT_DEBUG("[%s] remote_ptr: %" PRIx64 "\n", __func__, remote_ptr); + ggml_backend_buffer_t buffer = reinterpret_cast(remote_ptr); + void * base = ggml_backend_buffer_get_base(buffer); + // output serialization format: | base_ptr (8 bytes) | + uint64_t base_ptr = reinterpret_cast(base); + output.resize(sizeof(uint64_t), 0); + memcpy(output.data(), &base_ptr, sizeof(base_ptr)); +} + +static void rpc_free_buffer(const std::vector & input) { + // input serialization format: | remote_ptr (8 bytes) | + uint64_t remote_ptr; + memcpy(&remote_ptr, input.data(), sizeof(remote_ptr)); + GGML_PRINT_DEBUG("[%s] remote_ptr: %" PRIx64 "\n", __func__, remote_ptr); + ggml_backend_buffer_t buffer = reinterpret_cast(remote_ptr); + ggml_backend_buffer_free(buffer); +} + +static void rpc_buffer_clear(const std::vector & input) { + // input serialization format: | remote_ptr (8 bytes) | value (1 byte) | + uint64_t remote_ptr; + memcpy(&remote_ptr, input.data(), sizeof(remote_ptr)); + uint8_t value; + memcpy(&value, input.data() + sizeof(uint64_t), sizeof(value)); + GGML_PRINT_DEBUG("[%s] remote_ptr: %" PRIx64 ", value: %u\n", __func__, remote_ptr, value); + ggml_backend_buffer_t buffer = reinterpret_cast(remote_ptr); + ggml_backend_buffer_clear(buffer, value); +} + +static void rpc_set_tensor(const std::vector & input) { + // serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) | + const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); + uint64_t offset; + memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset)); + size_t size = input.size() - sizeof(rpc_tensor) - sizeof(offset); + + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + struct ggml_context * ctx = ggml_init(params); + ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor); + GGML_PRINT_DEBUG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu\n", __func__, (void*)tensor->buffer, tensor->data, offset, size); + const void * data = input.data() + sizeof(rpc_tensor) + sizeof(offset); + ggml_backend_tensor_set(tensor, data, offset, size); + ggml_free(ctx); +} + +static void rpc_get_tensor(const std::vector & input, std::vector & output) { + // serialization format: | rpc_tensor | offset (8 bytes) | size (8 bytes) | + const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); + uint64_t offset; + memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset)); + uint64_t size; + memcpy(&size, input.data() + sizeof(rpc_tensor) + sizeof(offset), sizeof(size)); + + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + struct ggml_context * ctx = ggml_init(params); + ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor); + GGML_PRINT_DEBUG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 "\n", __func__, (void*)tensor->buffer, tensor->data, offset, size); + // output serialization format: | data (size bytes) | + output.resize(size, 0); + ggml_backend_tensor_get(tensor, output.data(), offset, size); + ggml_free(ctx); +} + +static void rpc_copy_tensor(const std::vector & input, std::vector & output) { + // serialization format: | rpc_tensor src | rpc_tensor dst | + const rpc_tensor * rpc_src = (const rpc_tensor *)input.data(); + const rpc_tensor * rpc_dst = (const rpc_tensor *)(input.data() + sizeof(rpc_src)); + + struct ggml_init_params params { + /*.mem_size =*/ 2*ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + struct ggml_context * ctx = ggml_init(params); + ggml_tensor * src = deserialize_tensor(ctx, rpc_src); + ggml_tensor * dst = deserialize_tensor(ctx, rpc_dst); + GGML_PRINT_DEBUG("[%s] src->buffer: %p, dst->buffer: %p\n", __func__, (void*)src->buffer, (void*)dst->buffer); + bool result = ggml_backend_buffer_copy_tensor(src, dst); + // output serialization format: | result (1 byte) | + output.resize(1, 0); + output[0] = result; + ggml_free(ctx); +} + +static struct ggml_tensor * create_node(uint64_t id, + struct ggml_context * ctx, + const std::unordered_map & tensor_ptrs, + std::unordered_map & tensor_map) { + if (id == 0) { + return nullptr; + } + if (tensor_map.find(id) != tensor_map.end()) { + return tensor_map[id]; + } + const rpc_tensor * tensor = tensor_ptrs.at(id); + struct ggml_tensor * result = deserialize_tensor(ctx, tensor); + tensor_map[id] = result; + for (int i = 0; i < GGML_MAX_SRC; i++) { + result->src[i] = create_node(tensor->src[i], ctx, tensor_ptrs, tensor_map); + } + result->view_src = create_node(tensor->view_src, ctx, tensor_ptrs, tensor_map); + result->view_offs = tensor->view_offs; + return result; +} + +static void rpc_graph_compute(ggml_backend_t backend, const std::vector & input, std::vector & output) { + // serialization format: + // | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | + uint32_t n_nodes; + memcpy(&n_nodes, input.data(), sizeof(n_nodes)); + const uint64_t * nodes = (const uint64_t *)(input.data() + sizeof(n_nodes)); + uint32_t n_tensors; + memcpy(&n_tensors, input.data() + sizeof(n_nodes) + n_nodes*sizeof(uint64_t), sizeof(n_tensors)); + const rpc_tensor * tensors = (const rpc_tensor *)(input.data() + sizeof(n_nodes) + n_nodes*sizeof(uint64_t) + sizeof(n_tensors)); + GGML_PRINT_DEBUG("[%s] n_nodes: %u, n_tensors: %u\n", __func__, n_nodes, n_tensors); + + static size_t buf_size = ggml_tensor_overhead()*(n_nodes + n_tensors) + ggml_graph_overhead_custom(n_nodes, false); + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + struct ggml_context * ctx = ggml_init(params); + struct ggml_cgraph * graph = ggml_new_graph_custom(ctx, n_nodes, false); + graph->n_nodes = n_nodes; + std::unordered_map tensor_ptrs; + for (uint32_t i = 0; i < n_tensors; i++) { + tensor_ptrs[tensors[i].id] = &tensors[i]; + } + std::unordered_map tensor_map; + for (uint32_t i = 0; i < n_nodes; i++) { + graph->nodes[i] = create_node(nodes[i], ctx, tensor_ptrs, tensor_map); + } + ggml_status status = ggml_backend_graph_compute(backend, graph); + // output serialization format: | status (1 byte) | + output.resize(1, 0); + output[0] = status; + ggml_free(ctx); +} + +static void rpc_serve_client(ggml_backend_t backend, sockfd_t sockfd, size_t free_mem, size_t total_mem) { + while (true) { + uint8_t cmd; + if (!recv_data(sockfd, &cmd, 1)) { + break; + } + std::vector input; + std::vector output; + uint64_t input_size; + if (!recv_data(sockfd, &input_size, sizeof(input_size))) { + break; + } + input.resize(input_size); + if (!recv_data(sockfd, input.data(), input_size)) { + break; + } + switch (cmd) { + case ALLOC_BUFFER: { + rpc_alloc_buffer(backend, input, output); + break; + } + case GET_ALIGNMENT: { + rpc_get_alignment(backend, output); + break; + } + case GET_MAX_SIZE: { + rpc_get_max_size(backend, output); + break; + } + case BUFFER_GET_BASE: { + rpc_buffer_get_base(input, output); + break; + } + case FREE_BUFFER: { + rpc_free_buffer(input); + break; + } + case BUFFER_CLEAR: { + rpc_buffer_clear(input); + break; + } + case SET_TENSOR: { + rpc_set_tensor(input); + break; + } + case GET_TENSOR: { + rpc_get_tensor(input, output); + break; + } + case COPY_TENSOR: { + rpc_copy_tensor(input, output); + break; + } + case GRAPH_COMPUTE: { + rpc_graph_compute(backend, input, output); + break; + } + case GET_DEVICE_MEMORY: { + // output serialization format: | free (8 bytes) | total (8 bytes) | + output.resize(2*sizeof(uint64_t), 0); + memcpy(output.data(), &free_mem, sizeof(free_mem)); + memcpy(output.data() + sizeof(uint64_t), &total_mem, sizeof(total_mem)); + break; + } + default: { + fprintf(stderr, "Unknown command: %d\n", cmd); + return; + } + } + uint64_t output_size = output.size(); + if (!send_data(sockfd, &output_size, sizeof(output_size))) { + break; + } + if (!send_data(sockfd, output.data(), output_size)) { + break; + } + } +} + +void start_rpc_server(ggml_backend_t backend, const char * endpoint, size_t free_mem, size_t total_mem) { + std::string host; + int port; + if (!parse_endpoint(endpoint, host, port)) { + return; + } +#ifdef _WIN32 + { + WSADATA wsaData; + int res = WSAStartup(MAKEWORD(2, 2), &wsaData); + if (res != 0) { + fprintf(stderr, "WSAStartup failed: %d\n", res); + return; + } + } +#endif + auto server_socket = create_server_socket(host.c_str(), port); + if (server_socket == nullptr) { + fprintf(stderr, "Failed to create server socket\n"); + return; + } + while (true) { + auto client_socket = socket_accept(server_socket->fd); + if (client_socket == nullptr) { + fprintf(stderr, "Failed to accept client connection\n"); + return; + } + printf("Accepted client connection, free_mem=%zu, total_mem=%zu\n", free_mem, total_mem); + rpc_serve_client(backend, client_socket->fd, free_mem, total_mem); + printf("Client connection closed\n"); + } +#ifdef _WIN32 + WSACleanup(); +#endif +} diff --git a/ggml-rpc.h b/ggml-rpc.h new file mode 100644 index 000000000..aa144832a --- /dev/null +++ b/ggml-rpc.h @@ -0,0 +1,24 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GGML_RPC_MAX_SERVERS 16 + +// backend API +GGML_API GGML_CALL ggml_backend_t ggml_backend_rpc_init(const char * endpoint); +GGML_API GGML_CALL bool ggml_backend_is_rpc(ggml_backend_t backend); + +GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint); + +GGML_API GGML_CALL void ggml_backend_rpc_get_device_memory(const char * endpoint, size_t * free, size_t * total); + +GGML_API GGML_CALL void start_rpc_server(ggml_backend_t backend, const char * endpoint, size_t free_mem, size_t total_mem); + +#ifdef __cplusplus +} +#endif diff --git a/llama.cpp b/llama.cpp index ad35e4a2e..7d26966e4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -7,6 +7,10 @@ #include "ggml-alloc.h" #include "ggml-backend.h" +#ifdef GGML_USE_RPC +# include "ggml-rpc.h" +#endif + #ifdef GGML_USE_CUDA # include "ggml-cuda.h" #elif defined(GGML_USE_CLBLAST) @@ -1685,91 +1689,6 @@ static ggml_backend_buffer_type_t llama_default_buffer_type_cpu(bool host_buffer GGML_UNUSED(host_buffer); } -static ggml_backend_buffer_type_t llama_default_buffer_type_offload(int gpu) { - ggml_backend_buffer_type_t buft = nullptr; - -#ifdef GGML_USE_METAL - buft = ggml_backend_metal_buffer_type(); -#elif defined(GGML_USE_CUDA) - buft = ggml_backend_cuda_buffer_type(gpu); -#elif defined(GGML_USE_VULKAN) - buft = ggml_backend_vk_buffer_type(gpu); -#elif defined(GGML_USE_SYCL) - buft = ggml_backend_sycl_buffer_type(gpu); -#elif defined(GGML_USE_CLBLAST) - buft = ggml_backend_opencl_buffer_type(); -#elif defined(GGML_USE_KOMPUTE) - buft = ggml_backend_kompute_buffer_type(gpu); - if (buft == nullptr) { - LLAMA_LOG_WARN("%s: cannot use GPU %d, check `vulkaninfo --summary`\n", __func__, gpu); - } -#endif - - if (buft == nullptr) { - buft = llama_default_buffer_type_cpu(true); - } - return buft; - - GGML_UNUSED(gpu); -} - -static ggml_backend_buffer_type_t llama_default_buffer_type_split(int fallback_gpu, const float * tensor_split) { - ggml_backend_buffer_type_t buft = nullptr; - -#ifdef GGML_USE_CUDA - if (ggml_backend_cuda_get_device_count() > 1) { - buft = ggml_backend_cuda_split_buffer_type(tensor_split); - } -#endif - -#ifdef GGML_USE_SYCL - if (ggml_backend_sycl_get_device_count() > 1) { - buft = ggml_backend_sycl_split_buffer_type(tensor_split); - } -#endif - - if (buft == nullptr) { - buft = llama_default_buffer_type_offload(fallback_gpu); - } - return buft; - - GGML_UNUSED(tensor_split); -} - -static size_t llama_get_device_count() { -#if defined(GGML_USE_CUDA) - return ggml_backend_cuda_get_device_count(); -#elif defined(GGML_USE_SYCL) - return ggml_backend_sycl_get_device_count(); -#elif defined(GGML_USE_VULKAN) - return ggml_backend_vk_get_device_count(); -#else - return 1; -#endif -} - -static size_t llama_get_device_memory(int device) { -#if defined(GGML_USE_CUDA) - size_t total; - size_t free; - ggml_backend_cuda_get_device_memory(device, &free, &total); - return free; -#elif defined(GGML_USE_SYCL) - size_t total; - size_t free; - ggml_backend_sycl_get_device_memory(device, &free, &total); - return free; -#elif defined(GGML_USE_VULKAN) - size_t total; - size_t free; - ggml_backend_vk_get_device_memory(device, &free, &total); - return free; -#else - return 1; - GGML_UNUSED(device); -#endif -} - // // globals // @@ -2210,6 +2129,8 @@ struct llama_model { int main_gpu; int n_gpu_layers; + std::vector rpc_servers; + // gguf metadata std::unordered_map gguf_kv; @@ -2353,6 +2274,104 @@ struct llama_context { #endif }; +static ggml_backend_buffer_type_t llama_default_buffer_type_offload(const llama_model & model, int gpu) { + ggml_backend_buffer_type_t buft = nullptr; + +#ifdef GGML_USE_RPC + std::string endpoint = model.rpc_servers[gpu]; + buft = ggml_backend_rpc_buffer_type(endpoint.c_str()); +#elif defined(GGML_USE_METAL) + buft = ggml_backend_metal_buffer_type(); +#elif defined(GGML_USE_CUDA) + buft = ggml_backend_cuda_buffer_type(gpu); +#elif defined(GGML_USE_VULKAN) + buft = ggml_backend_vk_buffer_type(gpu); +#elif defined(GGML_USE_SYCL) + buft = ggml_backend_sycl_buffer_type(gpu); +#elif defined(GGML_USE_CLBLAST) + buft = ggml_backend_opencl_buffer_type(); +#elif defined(GGML_USE_KOMPUTE) + buft = ggml_backend_kompute_buffer_type(gpu); + if (buft == nullptr) { + LLAMA_LOG_WARN("%s: cannot use GPU %d, check `vulkaninfo --summary`\n", __func__, gpu); + } +#endif + + if (buft == nullptr) { + buft = llama_default_buffer_type_cpu(true); + } + return buft; + GGML_UNUSED(model); + GGML_UNUSED(gpu); +} + +static ggml_backend_buffer_type_t llama_default_buffer_type_split(const llama_model & model, int fallback_gpu, const float * tensor_split) { + ggml_backend_buffer_type_t buft = nullptr; + +#ifdef GGML_USE_CUDA + if (ggml_backend_cuda_get_device_count() > 1) { + buft = ggml_backend_cuda_split_buffer_type(tensor_split); + } +#endif + +#ifdef GGML_USE_SYCL + if (ggml_backend_sycl_get_device_count() > 1) { + buft = ggml_backend_sycl_split_buffer_type(tensor_split); + } +#endif + + if (buft == nullptr) { + buft = llama_default_buffer_type_offload(model, fallback_gpu); + } + return buft; + + GGML_UNUSED(tensor_split); +} + +static size_t llama_get_device_count(const llama_model & model) { +#if defined(GGML_USE_RPC) + return model.rpc_servers.size(); +#elif defined(GGML_USE_CUDA) + return ggml_backend_cuda_get_device_count(); +#elif defined(GGML_USE_SYCL) + return ggml_backend_sycl_get_device_count(); +#elif defined(GGML_USE_VULKAN) + return ggml_backend_vk_get_device_count(); +#else + return 1; +#endif + GGML_UNUSED(model); +} + +static size_t llama_get_device_memory(const llama_model & model, int device) { +#if defined(GGML_USE_RPC) + size_t total; + size_t free; + std::string endpoint = model.rpc_servers[device]; + ggml_backend_rpc_get_device_memory(endpoint.c_str(), &free, &total); + return free; +#elif defined(GGML_USE_CUDA) + size_t total; + size_t free; + ggml_backend_cuda_get_device_memory(device, &free, &total); + return free; +#elif defined(GGML_USE_SYCL) + size_t total; + size_t free; + ggml_backend_sycl_get_device_memory(device, &free, &total); + return free; +#elif defined(GGML_USE_VULKAN) + size_t total; + size_t free; + ggml_backend_vk_get_device_memory(device, &free, &total); + return free; +#else + return 1; +#endif + GGML_UNUSED(model); + GGML_UNUSED(device); +} + // // kv cache helpers // @@ -4791,13 +4810,13 @@ static bool llm_load_tensors( if (split_mode == LLAMA_SPLIT_MODE_LAYER) { // calculate the split points - int device_count = llama_get_device_count(); + int device_count = llama_get_device_count(model); bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + device_count, [](float x) { return x == 0.0f; }); std::vector splits(device_count); if (all_zero) { // default split, by free memory for (int i = 0; i < device_count; ++i) { - splits[i] = llama_get_device_memory(i); + splits[i] = llama_get_device_memory(model, i); } } else { std::copy(tensor_split, tensor_split + device_count, splits.begin()); @@ -4817,35 +4836,35 @@ static bool llm_load_tensors( int act_gpu_layers = std::min(n_gpu_layers, (int)n_layer + 1); for (int64_t i = i_gpu_start; i < n_layer; ++i) { int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(i - i_gpu_start)/act_gpu_layers) - splits.begin(); - model.buft_layer[i] = llama_default_buffer_type_offload(layer_gpu); + model.buft_layer[i] = llama_default_buffer_type_offload(model, layer_gpu); } // assign the output layer if (n_gpu_layers > n_layer) { int layer_gpu = std::upper_bound(splits.begin(), splits.begin() + device_count, float(act_gpu_layers - 1)/act_gpu_layers) - splits.begin(); - model.buft_output = llama_default_buffer_type_offload(layer_gpu); + model.buft_output = llama_default_buffer_type_offload(model, layer_gpu); } else { model.buft_output = llama_default_buffer_type_cpu(true); } } else { ggml_backend_buffer_type_t split_buft; if (split_mode == LLAMA_SPLIT_MODE_ROW) { - split_buft = llama_default_buffer_type_split(main_gpu, tensor_split); + split_buft = llama_default_buffer_type_split(model, main_gpu, tensor_split); } else { // LLAMA_SPLIT_MODE_NONE or LLAMA_SPLIT_MODE_LAYER in backends where it is not supported - split_buft = llama_default_buffer_type_offload(main_gpu); + split_buft = llama_default_buffer_type_offload(model, main_gpu); } // assign the repeating layers for (int64_t i = i_gpu_start; i < n_layer; ++i) { model.buft_layer[i] = { split_buft, - llama_default_buffer_type_offload(main_gpu) + llama_default_buffer_type_offload(model, main_gpu) }; } // assign the output layer if (n_gpu_layers > n_layer) { model.buft_output = { split_buft, - llama_default_buffer_type_offload(main_gpu) + llama_default_buffer_type_offload(model, main_gpu) }; } else { model.buft_output = llama_default_buffer_type_cpu(true); @@ -15390,6 +15409,7 @@ struct llama_model_params llama_model_default_params() { /*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER, /*.main_gpu =*/ 0, /*.tensor_split =*/ nullptr, + /*.rpc_servers =*/ nullptr, /*.progress_callback =*/ nullptr, /*.progress_callback_user_data =*/ nullptr, /*.kv_overrides =*/ nullptr, @@ -15460,7 +15480,9 @@ struct llama_model_quantize_params llama_model_quantize_default_params() { } size_t llama_max_devices(void) { -#if defined(GGML_USE_METAL) +#if defined(GGML_USE_RPC) + return GGML_RPC_MAX_SERVERS; +#elif defined(GGML_USE_METAL) return 1; #elif defined(GGML_USE_CUDA) return GGML_CUDA_MAX_DEVICES; @@ -15483,7 +15505,7 @@ bool llama_supports_mlock(void) { bool llama_supports_gpu_offload(void) { #if defined(GGML_USE_CUDA) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_METAL) || defined(GGML_USE_VULKAN) || \ - defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) + defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_RPC) // Defined when llama.cpp is compiled with support for offloading model layers to GPU. return true; #else @@ -15546,7 +15568,17 @@ struct llama_model * llama_load_model_from_file( return true; }; } - + if (params.rpc_servers != nullptr) { + // split the servers set them into model->rpc_servers + std::string servers(params.rpc_servers); + size_t pos = 0; + while ((pos = servers.find(",")) != std::string::npos) { + std::string server = servers.substr(0, pos); + model->rpc_servers.push_back(server); + servers.erase(0, pos + 1); + } + model->rpc_servers.push_back(servers); + } int status = llama_model_load(path_model, *model, params); GGML_ASSERT(status <= 0); if (status < 0) { @@ -15693,7 +15725,17 @@ struct llama_context * llama_new_context_with_model( if (!hparams.vocab_only) { // initialize backends -#ifdef GGML_USE_METAL +#if defined(GGML_USE_RPC) + for (auto & server : model->rpc_servers) { + ggml_backend_t backend = ggml_backend_rpc_init(server.c_str()); + if (backend == nullptr) { + LLAMA_LOG_ERROR("%s: failed to connect RPC backend to %s\n", __func__, server.c_str()); + llama_free(ctx); + return nullptr; + } + ctx->backends.push_back(backend); + } +#elif defined(GGML_USE_METAL) if (model->n_gpu_layers > 0) { ctx->backend_metal = ggml_backend_metal_init(); if (ctx->backend_metal == nullptr) { @@ -15850,7 +15892,7 @@ struct llama_context * llama_new_context_with_model( // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary bool pipeline_parallel = - llama_get_device_count() > 1 && + llama_get_device_count(*model) > 1 && model->n_gpu_layers > (int)model->hparams.n_layer && model->split_mode == LLAMA_SPLIT_MODE_LAYER && params.offload_kqv; diff --git a/llama.h b/llama.h index 0b2e708d0..612e32c4e 100644 --- a/llama.h +++ b/llama.h @@ -242,6 +242,9 @@ extern "C" { // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() const float * tensor_split; + // comma separated list of RPC servers to use for offloading + const char * rpc_servers; + // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. // If the provided progress_callback returns true, model loading continues. // If it returns false, model loading is immediately aborted. From 1265c670fd8e41e1947352c96c5179adda97fb2c Mon Sep 17 00:00:00 2001 From: Brian Date: Tue, 14 May 2024 23:10:39 +1000 Subject: [PATCH 017/137] Revert "move ndk code to a new library (#6951)" (#7282) This reverts commit efc8f767c8c8c749a245dd96ad4e2f37c164b54c. --- examples/llama.android/app/build.gradle.kts | 25 +++++++++- .../src/main/cpp}/CMakeLists.txt | 2 +- .../src/main/cpp/llama-android.cpp | 28 +++++------ .../src/main/java/com/example/llama/Llm.kt} | 8 +-- .../java/com/example/llama/MainViewModel.kt | 13 +++-- examples/llama.android/build.gradle.kts | 1 - examples/llama.android/llama/.gitignore | 1 - .../llama.android/llama/consumer-rules.pro | 0 .../llama.android/llama/proguard-rules.pro | 21 -------- .../llama/cpp/ExampleInstrumentedTest.kt | 24 --------- .../llama/src/main/AndroidManifest.xml | 4 -- .../llama/src/main/cpp/CMakeLists.txt | 49 ------------------- .../java/android/llama/cpp/ExampleUnitTest.kt | 17 ------- examples/llama.android/settings.gradle.kts | 1 - 14 files changed, 49 insertions(+), 145 deletions(-) rename examples/llama.android/{llama => app/src/main/cpp}/CMakeLists.txt (98%) rename examples/llama.android/{llama => app}/src/main/cpp/llama-android.cpp (92%) rename examples/llama.android/{llama/src/main/java/android/llama/cpp/LLamaAndroid.kt => app/src/main/java/com/example/llama/Llm.kt} (97%) delete mode 100644 examples/llama.android/llama/.gitignore delete mode 100644 examples/llama.android/llama/consumer-rules.pro delete mode 100644 examples/llama.android/llama/proguard-rules.pro delete mode 100644 examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt delete mode 100644 examples/llama.android/llama/src/main/AndroidManifest.xml delete mode 100644 examples/llama.android/llama/src/main/cpp/CMakeLists.txt delete mode 100644 examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt diff --git a/examples/llama.android/app/build.gradle.kts b/examples/llama.android/app/build.gradle.kts index 8d1b37195..d42140efe 100644 --- a/examples/llama.android/app/build.gradle.kts +++ b/examples/llama.android/app/build.gradle.kts @@ -7,6 +7,8 @@ android { namespace = "com.example.llama" compileSdk = 34 + ndkVersion = "26.1.10909125" + defaultConfig { applicationId = "com.example.llama" minSdk = 33 @@ -18,6 +20,17 @@ android { vectorDrawables { useSupportLibrary = true } + ndk { + // Add NDK properties if wanted, e.g. + // abiFilters += listOf("arm64-v8a") + } + externalNativeBuild { + cmake { + arguments += "-DCMAKE_BUILD_TYPE=Release" + cppFlags += listOf() + arguments += listOf() + } + } } buildTypes { @@ -42,6 +55,17 @@ android { composeOptions { kotlinCompilerExtensionVersion = "1.5.1" } + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + } + } + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } + } } dependencies { @@ -54,7 +78,6 @@ dependencies { implementation("androidx.compose.ui:ui-graphics") implementation("androidx.compose.ui:ui-tooling-preview") implementation("androidx.compose.material3:material3") - implementation(project(":llama")) testImplementation("junit:junit:4.13.2") androidTestImplementation("androidx.test.ext:junit:1.1.5") androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") diff --git a/examples/llama.android/llama/CMakeLists.txt b/examples/llama.android/app/src/main/cpp/CMakeLists.txt similarity index 98% rename from examples/llama.android/llama/CMakeLists.txt rename to examples/llama.android/app/src/main/cpp/CMakeLists.txt index bb5738ae3..85139329a 100644 --- a/examples/llama.android/llama/CMakeLists.txt +++ b/examples/llama.android/app/src/main/cpp/CMakeLists.txt @@ -37,7 +37,7 @@ FetchContent_MakeAvailable(llama) # used in the AndroidManifest.xml file. add_library(${CMAKE_PROJECT_NAME} SHARED # List C/C++ source files with relative paths to this CMakeLists.txt. - llama-android.cpp) + llama-android.cpp) # Specifies libraries CMake should link to your target library. You # can link libraries from various origins, such as libraries defined in this diff --git a/examples/llama.android/llama/src/main/cpp/llama-android.cpp b/examples/llama.android/app/src/main/cpp/llama-android.cpp similarity index 92% rename from examples/llama.android/llama/src/main/cpp/llama-android.cpp rename to examples/llama.android/app/src/main/cpp/llama-android.cpp index 874158ef0..4af9de303 100644 --- a/examples/llama.android/llama/src/main/cpp/llama-android.cpp +++ b/examples/llama.android/app/src/main/cpp/llama-android.cpp @@ -81,7 +81,7 @@ static void log_callback(ggml_log_level level, const char * fmt, void * data) { extern "C" JNIEXPORT jlong JNICALL -Java_android_llama_cpp_LLamaAndroid_load_1model(JNIEnv *env, jobject, jstring filename) { +Java_com_example_llama_Llm_load_1model(JNIEnv *env, jobject, jstring filename) { llama_model_params model_params = llama_model_default_params(); auto path_to_model = env->GetStringUTFChars(filename, 0); @@ -101,13 +101,13 @@ Java_android_llama_cpp_LLamaAndroid_load_1model(JNIEnv *env, jobject, jstring fi extern "C" JNIEXPORT void JNICALL -Java_android_llama_cpp_LLamaAndroid_free_1model(JNIEnv *, jobject, jlong model) { +Java_com_example_llama_Llm_free_1model(JNIEnv *, jobject, jlong model) { llama_free_model(reinterpret_cast(model)); } extern "C" JNIEXPORT jlong JNICALL -Java_android_llama_cpp_LLamaAndroid_new_1context(JNIEnv *env, jobject, jlong jmodel) { +Java_com_example_llama_Llm_new_1context(JNIEnv *env, jobject, jlong jmodel) { auto model = reinterpret_cast(jmodel); if (!model) { @@ -139,25 +139,25 @@ Java_android_llama_cpp_LLamaAndroid_new_1context(JNIEnv *env, jobject, jlong jmo extern "C" JNIEXPORT void JNICALL -Java_android_llama_cpp_LLamaAndroid_free_1context(JNIEnv *, jobject, jlong context) { +Java_com_example_llama_Llm_free_1context(JNIEnv *, jobject, jlong context) { llama_free(reinterpret_cast(context)); } extern "C" JNIEXPORT void JNICALL -Java_android_llama_cpp_LLamaAndroid_backend_1free(JNIEnv *, jobject) { +Java_com_example_llama_Llm_backend_1free(JNIEnv *, jobject) { llama_backend_free(); } extern "C" JNIEXPORT void JNICALL -Java_android_llama_cpp_LLamaAndroid_log_1to_1android(JNIEnv *, jobject) { +Java_com_example_llama_Llm_log_1to_1android(JNIEnv *, jobject) { llama_log_set(log_callback, NULL); } extern "C" JNIEXPORT jstring JNICALL -Java_android_llama_cpp_LLamaAndroid_bench_1model( +Java_com_example_llama_Llm_bench_1model( JNIEnv *env, jobject, jlong context_pointer, @@ -271,13 +271,13 @@ Java_android_llama_cpp_LLamaAndroid_bench_1model( extern "C" JNIEXPORT void JNICALL -Java_android_llama_cpp_LLamaAndroid_free_1batch(JNIEnv *, jobject, jlong batch_pointer) { +Java_com_example_llama_Llm_free_1batch(JNIEnv *, jobject, jlong batch_pointer) { llama_batch_free(*reinterpret_cast(batch_pointer)); } extern "C" JNIEXPORT jlong JNICALL -Java_android_llama_cpp_LLamaAndroid_new_1batch(JNIEnv *, jobject, jint n_tokens, jint embd, jint n_seq_max) { +Java_com_example_llama_Llm_new_1batch(JNIEnv *, jobject, jint n_tokens, jint embd, jint n_seq_max) { // Source: Copy of llama.cpp:llama_batch_init but heap-allocated. @@ -313,19 +313,19 @@ Java_android_llama_cpp_LLamaAndroid_new_1batch(JNIEnv *, jobject, jint n_tokens, extern "C" JNIEXPORT void JNICALL -Java_android_llama_cpp_LLamaAndroid_backend_1init(JNIEnv *, jobject) { +Java_com_example_llama_Llm_backend_1init(JNIEnv *, jobject) { llama_backend_init(); } extern "C" JNIEXPORT jstring JNICALL -Java_android_llama_cpp_LLamaAndroid_system_1info(JNIEnv *env, jobject) { +Java_com_example_llama_Llm_system_1info(JNIEnv *env, jobject) { return env->NewStringUTF(llama_print_system_info()); } extern "C" JNIEXPORT jint JNICALL -Java_android_llama_cpp_LLamaAndroid_completion_1init( +Java_com_example_llama_Llm_completion_1init( JNIEnv *env, jobject, jlong context_pointer, @@ -376,7 +376,7 @@ Java_android_llama_cpp_LLamaAndroid_completion_1init( extern "C" JNIEXPORT jstring JNICALL -Java_android_llama_cpp_LLamaAndroid_completion_1loop( +Java_com_example_llama_Llm_completion_1loop( JNIEnv * env, jobject, jlong context_pointer, @@ -438,6 +438,6 @@ Java_android_llama_cpp_LLamaAndroid_completion_1loop( extern "C" JNIEXPORT void JNICALL -Java_android_llama_cpp_LLamaAndroid_kv_1cache_1clear(JNIEnv *, jobject, jlong context) { +Java_com_example_llama_Llm_kv_1cache_1clear(JNIEnv *, jobject, jlong context) { llama_kv_cache_clear(reinterpret_cast(context)); } diff --git a/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt b/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt similarity index 97% rename from examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt rename to examples/llama.android/app/src/main/java/com/example/llama/Llm.kt index 6c63e54e0..d86afee37 100644 --- a/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt +++ b/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt @@ -1,4 +1,4 @@ -package android.llama.cpp +package com.example.llama import android.util.Log import kotlinx.coroutines.CoroutineDispatcher @@ -10,7 +10,7 @@ import kotlinx.coroutines.withContext import java.util.concurrent.Executors import kotlin.concurrent.thread -class LLamaAndroid { +class Llm { private val tag: String? = this::class.simpleName private val threadLocalState: ThreadLocal = ThreadLocal.withInitial { State.Idle } @@ -165,8 +165,8 @@ class LLamaAndroid { } // Enforce only one instance of Llm. - private val _instance: LLamaAndroid = LLamaAndroid() + private val _instance: Llm = Llm() - fun instance(): LLamaAndroid = _instance + fun instance(): Llm = _instance } } diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt index 45ac29938..be95e2221 100644 --- a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt +++ b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt @@ -1,6 +1,5 @@ package com.example.llama -import android.llama.cpp.LLamaAndroid import android.util.Log import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -10,7 +9,7 @@ import androidx.lifecycle.viewModelScope import kotlinx.coroutines.flow.catch import kotlinx.coroutines.launch -class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instance()): ViewModel() { +class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { companion object { @JvmStatic private val NanosPerSecond = 1_000_000_000.0 @@ -29,7 +28,7 @@ class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instan viewModelScope.launch { try { - llamaAndroid.unload() + llm.unload() } catch (exc: IllegalStateException) { messages += exc.message!! } @@ -45,7 +44,7 @@ class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instan messages += "" viewModelScope.launch { - llamaAndroid.send(text) + llm.send(text) .catch { Log.e(tag, "send() failed", it) messages += it.message!! @@ -58,7 +57,7 @@ class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instan viewModelScope.launch { try { val start = System.nanoTime() - val warmupResult = llamaAndroid.bench(pp, tg, pl, nr) + val warmupResult = llm.bench(pp, tg, pl, nr) val end = System.nanoTime() messages += warmupResult @@ -71,7 +70,7 @@ class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instan return@launch } - messages += llamaAndroid.bench(512, 128, 1, 3) + messages += llm.bench(512, 128, 1, 3) } catch (exc: IllegalStateException) { Log.e(tag, "bench() failed", exc) messages += exc.message!! @@ -82,7 +81,7 @@ class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instan fun load(pathToModel: String) { viewModelScope.launch { try { - llamaAndroid.load(pathToModel) + llm.load(pathToModel) messages += "Loaded $pathToModel" } catch (exc: IllegalStateException) { Log.e(tag, "load() failed", exc) diff --git a/examples/llama.android/build.gradle.kts b/examples/llama.android/build.gradle.kts index acd1ada7d..50ebc8211 100644 --- a/examples/llama.android/build.gradle.kts +++ b/examples/llama.android/build.gradle.kts @@ -2,5 +2,4 @@ plugins { id("com.android.application") version "8.2.0" apply false id("org.jetbrains.kotlin.android") version "1.9.0" apply false - id("com.android.library") version "8.2.0" apply false } diff --git a/examples/llama.android/llama/.gitignore b/examples/llama.android/llama/.gitignore deleted file mode 100644 index 796b96d1c..000000000 --- a/examples/llama.android/llama/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build diff --git a/examples/llama.android/llama/consumer-rules.pro b/examples/llama.android/llama/consumer-rules.pro deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/llama.android/llama/proguard-rules.pro b/examples/llama.android/llama/proguard-rules.pro deleted file mode 100644 index f1b424510..000000000 --- a/examples/llama.android/llama/proguard-rules.pro +++ /dev/null @@ -1,21 +0,0 @@ -# Add project specific ProGuard rules here. -# You can control the set of applied configuration files using the -# proguardFiles setting in build.gradle. -# -# For more details, see -# http://developer.android.com/guide/developing/tools/proguard.html - -# If your project uses WebView with JS, uncomment the following -# and specify the fully qualified class name to the JavaScript interface -# class: -#-keepclassmembers class fqcn.of.javascript.interface.for.webview { -# public *; -#} - -# Uncomment this to preserve the line number information for -# debugging stack traces. -#-keepattributes SourceFile,LineNumberTable - -# If you keep the line number information, uncomment this to -# hide the original source file name. -#-renamesourcefileattribute SourceFile diff --git a/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt b/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt deleted file mode 100644 index 05d6ab5d2..000000000 --- a/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt +++ /dev/null @@ -1,24 +0,0 @@ -package android.llama.cpp - -import androidx.test.platform.app.InstrumentationRegistry -import androidx.test.ext.junit.runners.AndroidJUnit4 - -import org.junit.Test -import org.junit.runner.RunWith - -import org.junit.Assert.* - -/** - * Instrumented test, which will execute on an Android device. - * - * See [testing documentation](http://d.android.com/tools/testing). - */ -@RunWith(AndroidJUnit4::class) -class ExampleInstrumentedTest { - @Test - fun useAppContext() { - // Context of the app under test. - val appContext = InstrumentationRegistry.getInstrumentation().targetContext - assertEquals("android.llama.cpp.test", appContext.packageName) - } -} diff --git a/examples/llama.android/llama/src/main/AndroidManifest.xml b/examples/llama.android/llama/src/main/AndroidManifest.xml deleted file mode 100644 index 8bdb7e14b..000000000 --- a/examples/llama.android/llama/src/main/AndroidManifest.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/examples/llama.android/llama/src/main/cpp/CMakeLists.txt b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt deleted file mode 100644 index 42ebaad49..000000000 --- a/examples/llama.android/llama/src/main/cpp/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -# For more information about using CMake with Android Studio, read the -# documentation: https://d.android.com/studio/projects/add-native-code.html. -# For more examples on how to use CMake, see https://github.com/android/ndk-samples. - -# Sets the minimum CMake version required for this project. -cmake_minimum_required(VERSION 3.22.1) - -# Declares the project name. The project name can be accessed via ${ PROJECT_NAME}, -# Since this is the top level CMakeLists.txt, the project name is also accessible -# with ${CMAKE_PROJECT_NAME} (both CMake variables are in-sync within the top level -# build script scope). -project("llama-android") - -include(FetchContent) -FetchContent_Declare( - llama - GIT_REPOSITORY https://github.com/ggerganov/llama.cpp - GIT_TAG master -) - -# Also provides "common" -FetchContent_MakeAvailable(llama) - -# Creates and names a library, sets it as either STATIC -# or SHARED, and provides the relative paths to its source code. -# You can define multiple libraries, and CMake builds them for you. -# Gradle automatically packages shared libraries with your APK. -# -# In this top level CMakeLists.txt, ${CMAKE_PROJECT_NAME} is used to define -# the target library name; in the sub-module's CMakeLists.txt, ${PROJECT_NAME} -# is preferred for the same purpose. -# -# In order to load a library into your app from Java/Kotlin, you must call -# System.loadLibrary() and pass the name of the library defined here; -# for GameActivity/NativeActivity derived applications, the same library name must be -# used in the AndroidManifest.xml file. -add_library(${CMAKE_PROJECT_NAME} SHARED - # List C/C++ source files with relative paths to this CMakeLists.txt. - llama-android.cpp) - -# Specifies libraries CMake should link to your target library. You -# can link libraries from various origins, such as libraries defined in this -# build script, prebuilt third-party libraries, or Android system libraries. -target_link_libraries(${CMAKE_PROJECT_NAME} - # List libraries link to the target library - llama - common - android - log) diff --git a/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt b/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt deleted file mode 100644 index cbbb974d3..000000000 --- a/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt +++ /dev/null @@ -1,17 +0,0 @@ -package android.llama.cpp - -import org.junit.Test - -import org.junit.Assert.* - -/** - * Example local unit test, which will execute on the development machine (host). - * - * See [testing documentation](http://d.android.com/tools/testing). - */ -class ExampleUnitTest { - @Test - fun addition_isCorrect() { - assertEquals(4, 2 + 2) - } -} diff --git a/examples/llama.android/settings.gradle.kts b/examples/llama.android/settings.gradle.kts index c7c1a034a..2ba32c4fa 100644 --- a/examples/llama.android/settings.gradle.kts +++ b/examples/llama.android/settings.gradle.kts @@ -15,4 +15,3 @@ dependencyResolutionManagement { rootProject.name = "LlamaAndroid" include(":app") -include(":llama") From 4f0263633b40e94e8b69fd6e7e4395cfedfd5c12 Mon Sep 17 00:00:00 2001 From: Steve Grubb Date: Tue, 14 May 2024 10:11:24 -0400 Subject: [PATCH 018/137] server: free sampling contexts on exit (#7264) * server: free sampling contexts on exit This cleans up last leak found by the address sanitizer. * fix whitespace * fix whitespace --- examples/server/server.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index ceaeb1f76..7e0d068f8 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -671,6 +671,13 @@ struct server_context { model = nullptr; } + // Clear any sampling context + for (server_slot & slot : slots) { + if (slot.ctx_sampling != nullptr) { + llama_sampling_free(slot.ctx_sampling); + } + } + llama_batch_free(batch); } From 0d26d8ccd8caebab75af697c0275f599075fdacf Mon Sep 17 00:00:00 2001 From: Hong Bo PENG Date: Sun, 12 May 2024 17:17:18 +0800 Subject: [PATCH 019/137] ggml : optimize for ppc64le using VSX intrinsics (ggml/784) * optimize for ppc64le using VSX intrinsics * 1. code clean up by removing comments about overflow concern. 2. fix typo in suffix of scaling. * Continue to fix typo in suffix of scaling for QK_K <> 256 --------- Co-authored-by: Georgi Gerganov --- ggml-quants.c | 2169 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 2167 insertions(+), 2 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 00334c5fe..f711bd013 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -241,7 +241,7 @@ static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 #endif // __AVX__ || __AVX2__ || __AVX512F__ #endif // defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__SSSE3__) -#if defined(__ARM_NEON) || defined(__wasm_simd128__) +#if defined(__ARM_NEON) || defined(__wasm_simd128__) || defined(__POWER9_VECTOR__) #define B1(c,s,n) 0x ## n ## c , 0x ## n ## s #define B2(c,s,n) B1(c,s,n ## c), B1(c,s,n ## s) #define B3(c,s,n) B2(c,s,n ## c), B2(c,s,n ## s) @@ -643,6 +643,38 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) // store result __riscv_vse8_v_i8m1(y[i].qs , vs, vl); } +#elif defined(__POWER9_VECTOR__) + for (int i = 0; i < nb; i++) { + vector float srcv [8]; + vector float asrcv[8]; + vector float amaxv[8]; + vector signed int vi[8]; + + for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j); + for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]); + + for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]); + for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]); + for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]); + + const float amax = MAX(MAX(vec_extract(amaxv[0], 0), + vec_extract(amaxv[0], 1)), + MAX(vec_extract(amaxv[0], 2), + vec_extract(amaxv[0], 3))); + + const float d = amax / ((1 << 7) - 1); + const float id = d ? 1.0f/d : 0.0f; + const vector float vid = vec_splats(id); + + y[i].d = GGML_FP32_TO_FP16(d); + + for (int j = 0; j < 8; j++) { + const vector float v = vec_round(vec_mul(srcv[j], vid)); + vi[j] = vec_cts(v, 0); + } + vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]); + vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]); + } #else GGML_UNUSED(nb); // scalar @@ -898,6 +930,46 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) int sum = __riscv_vmv_x_s_i16m1_i16(vwrs); y[i].s = GGML_FP32_TO_FP16(sum*d); } +#elif defined(__POWER9_VECTOR__) + for (int i = 0; i < nb; i++) { + vector float srcv [8]; + vector float asrcv[8]; + vector float amaxv[8]; + vector signed int vi[8]; + + for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j); + for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]); + + for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]); + for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]); + for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]); + + const float amax = MAX(MAX(vec_extract(amaxv[0], 0), + vec_extract(amaxv[0], 1)), + MAX(vec_extract(amaxv[0], 2), + vec_extract(amaxv[0], 3))); + + const float d = amax / ((1 << 7) - 1); + const float id = d ? 1.0f/d : 0.0f; + const vector float vid = vec_splats(id); + + y[i].d = GGML_FP32_TO_FP16(d); + + vector int accv = vec_splats(0); + + for (int j = 0; j < 8; j++) { + const vector float v = vec_round(vec_mul(srcv[j], vid)); + vi[j] = vec_cts(v, 0); + + accv = vec_add(accv, vi[j]); + } + vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]); + vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]); + + accv = vec_add(accv, vec_sld(accv, accv, 4)); + accv = vec_add(accv, vec_sld(accv, accv, 8)); + y[i].s = GGML_FP32_TO_FP16(d * vec_extract(accv, 0)); + } #else GGML_UNUSED(nb); // scalar @@ -3740,6 +3812,46 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r } *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + const vector signed char v8 = vec_splats((signed char)0x8); + + vector float vsumf0 = vec_splats(0.0f); + +#pragma GCC unroll 4 + for (int i = 0; i < nb; i++) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d)); + vector float vd = vec_mul(vxd, vyd); + + vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs); + vector signed char q8y0 = vec_xl( 0, y[i].qs); + vector signed char q8y1 = vec_xl(16, y[i].qs); + + vector signed char q4x0 = vec_and(qxs, lowMask); + vector signed char q4x1 = vec_sr(qxs, v4); + + q4x0 = vec_sub(q4x0, v8); + q4x1 = vec_sub(q4x1, v8); + + vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1)); + + qv0 = vec_add(qv0, qv1); + + vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + } + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else // scalar float sumf = 0.0; @@ -3958,6 +4070,46 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r } *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + +#pragma GCC unroll 4 + for (int i = 0; i < nb; i++) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d)); + vector float vd = vec_mul(vxd, vyd); + + vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].m)); + vector float vys = {GGML_FP16_TO_FP32(y[i].s), 0.0f, 0.0f, 0.0f}; + vsumf0 = vec_madd(vxmin, vys, vsumf0); + + vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs); + vector signed char q8y0 = vec_xl( 0, y[i].qs); + vector signed char q8y1 = vec_xl(16, y[i].qs); + + vector signed char q4x0 = vec_and(qxs, lowMask); + vector signed char q4x1 = vec_sr(qxs, v4); + + vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1)); + + qv0 = vec_add(qv0, qv1); + + vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + } + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else // scalar float sumf = 0.0; @@ -4243,6 +4395,49 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r } *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v4 = vec_splats((unsigned char)4); + + vector float vsumf0 = vec_splats(0.0f); + +#pragma GCC unroll 4 + for (int i = 0; i < nb; ++i) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d)); + vector float vd = vec_mul(vxd, vyd); + + vector signed long long aux64x2_0 = {(uint64_t)(table_b2b_1[x[i].qh[0]]), (uint64_t)(table_b2b_1[x[i].qh[1]])}; + vector signed long long aux64x2_1 = {(uint64_t)(table_b2b_1[x[i].qh[2]]), (uint64_t)(table_b2b_1[x[i].qh[3]])}; + + vector signed char qh0 = (vector signed char)aux64x2_0; + vector signed char qh1 = (vector signed char)aux64x2_1; + + vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs); + + vector signed char q5x0 = vec_sub(vec_and (qxs, lowMask), qh0); + vector signed char q5x1 = vec_sub(vec_sr(qxs, v4), qh1); + + vector signed char q8y0 = vec_xl( 0, y[i].qs); + vector signed char q8y1 = vec_xl( 16, y[i].qs); + + vector signed short qv0 = vec_add(vec_mule(q5x0, q8y0), vec_mulo(q5x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q5x1, q8y1), vec_mulo(q5x1, q8y1)); + + qv0 = vec_add(qv0, qv1); + + vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + } + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else // scalar float sumf = 0.0; @@ -4547,6 +4742,53 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r } *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + +#pragma GCC unroll 4 + for (int i = 0; i < nb; ++i) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d)); + vector float vd = vec_mul(vxd, vyd); + + vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].m)); + vector float vys = {GGML_FP16_TO_FP32(y[i].s), 0.f, 0.f, 0.f}; + vsumf0 = vec_madd(vxmin, vys, vsumf0); + + vector unsigned long long aux64x2_0 = {(uint64_t)(table_b2b_0[x[i].qh[0]]), (uint64_t)(table_b2b_0[x[i].qh[1]])}; + vector unsigned long long aux64x2_1 = {(uint64_t)(table_b2b_0[x[i].qh[2]]), (uint64_t)(table_b2b_0[x[i].qh[3]])}; + + vector signed char qh0 = (vector signed char)aux64x2_0; + vector signed char qh1 = (vector signed char)aux64x2_1; + + vector signed char qxs = (vector signed char)vec_xl( 0, x[i].qs); + + vector signed char q5x0 = vec_or(vec_and(qxs, lowMask), qh0); + vector signed char q5x1 = vec_or(vec_sr(qxs, v4), qh1); + + vector signed char q8y0 = vec_xl( 0, y[i].qs); + vector signed char q8y1 = vec_xl( 16, y[i].qs); + + vector signed short qv0 = vec_add(vec_mule(q5x0, q8y0), vec_mulo(q5x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q5x1, q8y1), vec_mulo(q5x1, q8y1)); + + qv0 = vec_add(qv0, qv1); + + vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + } + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else // scalar float sumf = 0.0; @@ -4722,6 +4964,45 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r } *s = sumf; +#elif defined(__POWER9_VECTOR__) + vector float vsumf0 = vec_splats(0.0f); + +#pragma GCC unroll 4 + for (int i = 0; i < nb; i++) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[i].d)); + vector float vd = vec_mul(vxd, vyd); + + vector signed char q8x0 = vec_xl( 0, x[i].qs); + vector signed char q8x1 = vec_xl(16, x[i].qs); + vector signed char q8y0 = vec_xl( 0, y[i].qs); + vector signed char q8y1 = vec_xl(16, y[i].qs); + + vector signed short qv0 = vec_mule(q8x0, q8y0); + vector signed short qv1 = vec_mulo(q8x0, q8y0); + vector signed short qv2 = vec_mule(q8x1, q8y1); + vector signed short qv3 = vec_mulo(q8x1, q8y1); + + vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackh(qv1)); + vector signed int vsumi1 = vec_add(vec_unpackl(qv0), vec_unpackl(qv1)); + vector signed int vsumi2 = vec_add(vec_unpackh(qv2), vec_unpackh(qv3)); + vector signed int vsumi3 = vec_add(vec_unpackl(qv2), vec_unpackl(qv3)); + + vsumi0 = vec_add(vsumi0, vsumi2); + vsumi1 = vec_add(vsumi1, vsumi3); + + vsumi0 = vec_add(vsumi0, vsumi1); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + } + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else // scalar float sumf = 0.0; @@ -5077,6 +5358,147 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0x3); + const vector signed char lowScaleMask = vec_splats((signed char)0xF); + const vector unsigned char v2 = vec_splats((unsigned char)0x2); + const vector unsigned char v6 = vec_splats((unsigned char)0x6); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin)); + vector float vdmin = vec_mul(vxmin, vyd); + + vector signed short q8ysums0 = vec_xl( 0, y[i].bsums); + vector signed short q8ysums1 = vec_xl(16, y[i].bsums); + + vector signed char q2xmins = (vector signed char)vec_xl( 0, x[i].scales); + vector signed char vscales = vec_and(q2xmins, lowScaleMask); + + q2xmins = vec_sr(q2xmins, v4); + vector signed short q2xmins0 = vec_unpackh(q2xmins); + vector signed short q2xmins1 = vec_unpackl(q2xmins); + + vector signed int prod0 = vec_mule(q2xmins0, q8ysums0); + vector signed int prod1 = vec_mulo(q2xmins0, q8ysums0); + vector signed int prod2 = vec_mule(q2xmins1, q8ysums1); + vector signed int prod3 = vec_mulo(q2xmins1, q8ysums1); + + vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0); + vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1); + vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2); + vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + const uint8_t * restrict q2 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/128; ++j) { + __builtin_prefetch(q2, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, q2); + vector signed char qxs1 = (vector signed char)vec_xl(16, q2); + q2 += 32; + + vector signed char q2x00 = vec_and(qxs0, lowMask); + vector signed char q2x01 = vec_and(vec_sr(qxs0, v2), lowMask); + vector signed char q2x02 = vec_and(vec_sr(qxs0, v4), lowMask); + vector signed char q2x03 = vec_and(vec_sr(qxs0, v6), lowMask); + vector signed char q2x10 = vec_and(qxs1, lowMask); + vector signed char q2x11 = vec_and(vec_sr(qxs1, v2), lowMask); + vector signed char q2x12 = vec_and(vec_sr(qxs1, v4), lowMask); + vector signed char q2x13 = vec_and(vec_sr(qxs1, v6), lowMask); + + vector signed char q8y00 = vec_xl( 0, q8); + vector signed char q8y10 = vec_xl( 16, q8); + vector signed char q8y01 = vec_xl( 32, q8); + vector signed char q8y11 = vec_xl( 48, q8); + vector signed char q8y02 = vec_xl( 64, q8); + vector signed char q8y12 = vec_xl( 80, q8); + vector signed char q8y03 = vec_xl( 96, q8); + vector signed char q8y13 = vec_xl(112, q8); + q8 += 128; + + vector signed short qv0 = vec_add(vec_mule(q2x00, q8y00), vec_mulo(q2x00, q8y00)); + vector signed short qv1 = vec_add(vec_mule(q2x01, q8y01), vec_mulo(q2x01, q8y01)); + vector signed short qv2 = vec_add(vec_mule(q2x02, q8y02), vec_mulo(q2x02, q8y02)); + vector signed short qv3 = vec_add(vec_mule(q2x03, q8y03), vec_mulo(q2x03, q8y03)); + vector signed short qv4 = vec_add(vec_mule(q2x10, q8y10), vec_mulo(q2x10, q8y10)); + vector signed short qv5 = vec_add(vec_mule(q2x11, q8y11), vec_mulo(q2x11, q8y11)); + vector signed short qv6 = vec_add(vec_mule(q2x12, q8y12), vec_mulo(q2x12, q8y12)); + vector signed short qv7 = vec_add(vec_mule(q2x13, q8y13), vec_mulo(q2x13, q8y13)); + + vector signed short vscales_h = vec_unpackh(vscales); + vector signed short vs0 = vec_splat(vscales_h, 0); + vector signed short vs1 = vec_splat(vscales_h, 1); + vector signed short vs2 = vec_splat(vscales_h, 2); + vector signed short vs3 = vec_splat(vscales_h, 3); + vector signed short vs4 = vec_splat(vscales_h, 4); + vector signed short vs5 = vec_splat(vscales_h, 5); + vector signed short vs6 = vec_splat(vscales_h, 6); + vector signed short vs7 = vec_splat(vscales_h, 7); + vscales = vec_sld(vscales, vscales, 8); + + qv0 = vec_mul(qv0, vs0); + qv1 = vec_mul(qv1, vs2); + qv2 = vec_mul(qv2, vs4); + qv3 = vec_mul(qv3, vs6); + + qv0 = vec_madd(qv4, vs1, qv0); + qv1 = vec_madd(qv5, vs3, qv1); + qv2 = vec_madd(qv6, vs5, qv2); + qv3 = vec_madd(qv7, vs7, qv3); + + vsumi0 = vec_add(vec_unpackh(qv0), vsumi0); + vsumi1 = vec_add(vec_unpackh(qv1), vsumi1); + vsumi2 = vec_add(vec_unpackh(qv2), vsumi2); + vsumi3 = vec_add(vec_unpackh(qv3), vsumi3); + + vsumi4 = vec_add(vec_unpackl(qv0), vsumi4); + vsumi5 = vec_add(vec_unpackl(qv1), vsumi5); + vsumi6 = vec_add(vec_unpackl(qv2), vsumi6); + vsumi7 = vec_add(vec_unpackl(qv3), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); + #else float sumf = 0; @@ -5347,6 +5769,87 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0x3); + const vector signed char lowScaleMask = vec_splats((signed char)0xF); + const vector unsigned char v2 = vec_splats((unsigned char)0x2); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + const vector unsigned char v6 = vec_splats((unsigned char)0x6); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + +#pragma GCC unroll 2 + for (int i = 0; i < nb; ++i) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin)); + vector float vdmin = vec_mul(vxmin, vyd); + + vector signed short q8ysums0 = vec_xl_len(y[i].bsums, 8); + + vector signed char q2xmins = (vector signed char)vec_xl_len(x[i].scales, 4); + vector signed char vscales = vec_and(q2xmins, lowScaleMask); + + q2xmins = vec_sr(q2xmins, v4); + vector signed short q2xmins0 = vec_unpackh((vector signed char)q2xmins); + + vector signed int prod0 = vec_mule(q2xmins0, q8ysums0); + vector signed int prod1 = vec_mulo(q2xmins0, q8ysums0); + + vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0); + vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs); + vector signed char q2x00 = vec_and(qxs0, lowMask); + vector signed char q2x01 = vec_and(vec_sr(qxs0, v2), lowMask); + vector signed char q2x02 = vec_and(vec_sr(qxs0, v4), lowMask); + vector signed char q2x03 = vec_and(vec_sr(qxs0, v6), lowMask); + + vector signed char q8y00 = vec_xl( 0, y[i].qs); + vector signed char q8y01 = vec_xl( 16, y[i].qs); + vector signed char q8y02 = vec_xl( 32, y[i].qs); + vector signed char q8y03 = vec_xl( 48, y[i].qs); + + vector signed short qv0 = vec_add(vec_mule(q2x00, q8y00), vec_mulo(q2x00, q8y00)); + vector signed short qv1 = vec_add(vec_mule(q2x01, q8y01), vec_mulo(q2x01, q8y01)); + vector signed short qv2 = vec_add(vec_mule(q2x02, q8y02), vec_mulo(q2x02, q8y02)); + vector signed short qv3 = vec_add(vec_mule(q2x03, q8y03), vec_mulo(q2x03, q8y03)); + + vector signed short vscales_h = vec_unpackh(vscales); + vector signed short vs0 = vec_splat(vscales_h, 0); + vector signed short vs1 = vec_splat(vscales_h, 1); + vector signed short vs2 = vec_splat(vscales_h, 2); + vector signed short vs3 = vec_splat(vscales_h, 3); + + vector signed int vsumi0 = vec_add(vec_mule(qv0, vs0), vec_mulo(qv0, vs0)); + vector signed int vsumi1 = vec_add(vec_mule(qv1, vs1), vec_mulo(qv1, vs1)); + vector signed int vsumi2 = vec_add(vec_mule(qv2, vs2), vec_mulo(qv2, vs2)); + vector signed int vsumi3 = vec_add(vec_mule(qv3, vs3), vec_mulo(qv3, vs3)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); + #else float sumf = 0; @@ -5841,6 +6344,160 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0x3); + const vector signed char v1 = vec_splats((signed char)0x1); + const vector unsigned char v2 = vec_splats((unsigned char)0x2); + const vector unsigned char v3 = vec_splats((unsigned char)0x3); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + const vector unsigned char v6 = vec_splats((unsigned char)0x6); + const vector signed char off = vec_splats((signed char)0x20); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + uint32_t aux[3]; + uint32_t utmp[4]; + + memcpy(aux, x[i].scales, 12); + utmp[3] = ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4); + utmp[2] = ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4); + utmp[1] = (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4); + utmp[0] = (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4); + + vector signed char vscales = (vector signed char)vec_xl( 0, utmp); + vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].hmask); + vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].hmask); + + vscales = vec_sub(vscales, off); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + const uint8_t * restrict q3 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/128; ++j) { + __builtin_prefetch(q3, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, q3); + vector signed char qxs1 = (vector signed char)vec_xl(16, q3); + q3 += 32; + + //the low 2 bits + vector signed char qxs00 = vec_and(qxs0, lowMask); + vector signed char qxs01 = vec_and(vec_sr(qxs0, v2), lowMask); + vector signed char qxs02 = vec_and(vec_sr(qxs0, v4), lowMask); + vector signed char qxs03 = vec_and(vec_sr(qxs0, v6), lowMask); + vector signed char qxs10 = vec_and(qxs1, lowMask); + vector signed char qxs11 = vec_and(vec_sr(qxs1, v2), lowMask); + vector signed char qxs12 = vec_and(vec_sr(qxs1, v4), lowMask); + vector signed char qxs13 = vec_and(vec_sr(qxs1, v6), lowMask); + + //the 3rd bit + vector signed char qxh00 = vec_sl(vec_andc(v1, qxhs0), v2); + vector signed char qxh01 = vec_sl(vec_andc(v1, vec_sr(qxhs0, (vector unsigned char)v1)), v2); + vector signed char qxh02 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v2)), v2); + vector signed char qxh03 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v3)), v2); + vector signed char qxh10 = vec_sl(vec_andc(v1, qxhs1), v2); + vector signed char qxh11 = vec_sl(vec_andc(v1, vec_sr(qxhs1, (vector unsigned char)v1)), v2); + vector signed char qxh12 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v2)), v2); + vector signed char qxh13 = vec_sl(vec_andc(v1, vec_sr(qxhs1, v3)), v2); + qxhs0 = vec_sr(qxhs0, v4); + qxhs1 = vec_sr(qxhs1, v4); + + vector signed char q3x00 = vec_sub(qxs00, qxh00); + vector signed char q3x01 = vec_sub(qxs01, qxh01); + vector signed char q3x02 = vec_sub(qxs02, qxh02); + vector signed char q3x03 = vec_sub(qxs03, qxh03); + vector signed char q3x10 = vec_sub(qxs10, qxh10); + vector signed char q3x11 = vec_sub(qxs11, qxh11); + vector signed char q3x12 = vec_sub(qxs12, qxh12); + vector signed char q3x13 = vec_sub(qxs13, qxh13); + + vector signed char q8y00 = vec_xl( 0, q8); + vector signed char q8y10 = vec_xl( 16, q8); + vector signed char q8y01 = vec_xl( 32, q8); + vector signed char q8y11 = vec_xl( 48, q8); + vector signed char q8y02 = vec_xl( 64, q8); + vector signed char q8y12 = vec_xl( 80, q8); + vector signed char q8y03 = vec_xl( 96, q8); + vector signed char q8y13 = vec_xl(112, q8); + q8 += 128; + + vector signed short vscales_h = vec_unpackh(vscales); + vector signed short vs0 = vec_splat(vscales_h, 0); + vector signed short vs1 = vec_splat(vscales_h, 1); + vector signed short vs2 = vec_splat(vscales_h, 2); + vector signed short vs3 = vec_splat(vscales_h, 3); + vector signed short vs4 = vec_splat(vscales_h, 4); + vector signed short vs5 = vec_splat(vscales_h, 5); + vector signed short vs6 = vec_splat(vscales_h, 6); + vector signed short vs7 = vec_splat(vscales_h, 7); + vscales = vec_sld(vscales, vscales, 8); + + vector signed short qv00 = vec_add(vec_mule(q3x00, q8y00), vec_mulo(q3x00, q8y00)); + vector signed short qv01 = vec_add(vec_mule(q3x01, q8y01), vec_mulo(q3x01, q8y01)); + vector signed short qv02 = vec_add(vec_mule(q3x02, q8y02), vec_mulo(q3x02, q8y02)); + vector signed short qv03 = vec_add(vec_mule(q3x03, q8y03), vec_mulo(q3x03, q8y03)); + vector signed short qv10 = vec_add(vec_mule(q3x10, q8y10), vec_mulo(q3x10, q8y10)); + vector signed short qv11 = vec_add(vec_mule(q3x11, q8y11), vec_mulo(q3x11, q8y11)); + vector signed short qv12 = vec_add(vec_mule(q3x12, q8y12), vec_mulo(q3x12, q8y12)); + vector signed short qv13 = vec_add(vec_mule(q3x13, q8y13), vec_mulo(q3x13, q8y13)); + + vector signed int vsum0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); + vector signed int vsum1 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2)); + vector signed int vsum2 = vec_add(vec_mule(qv02, vs4), vec_mulo(qv02, vs4)); + vector signed int vsum3 = vec_add(vec_mule(qv03, vs6), vec_mulo(qv03, vs6)); + vector signed int vsum4 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1)); + vector signed int vsum5 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3)); + vector signed int vsum6 = vec_add(vec_mule(qv12, vs5), vec_mulo(qv12, vs5)); + vector signed int vsum7 = vec_add(vec_mule(qv13, vs7), vec_mulo(qv13, vs7)); + + vsumi0 = vec_add(vsum0, vsumi0); + vsumi1 = vec_add(vsum1, vsumi1); + vsumi2 = vec_add(vsum2, vsumi2); + vsumi3 = vec_add(vsum3, vsumi3); + vsumi4 = vec_add(vsum4, vsumi4); + vsumi5 = vec_add(vsum5, vsumi5); + vsumi6 = vec_add(vsum6, vsumi6); + vsumi7 = vec_add(vsum7, vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else // scalar version // This function is written like this so the compiler can manage to vectorize most of it @@ -6207,6 +6864,95 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0x3); + const vector signed char v1 = vec_splats((signed char)0x1); + const vector unsigned char v2 = vec_splats((unsigned char)0x2); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + const vector unsigned char v6 = vec_splats((unsigned char)0x6); + const vector signed char off = vec_splats((signed char)0x8); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + +#pragma GCC unroll 2 + for (int i = 0; i < nb; ++i) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + uint16_t aux16[2]; + int8_t * scales = (int8_t *)aux16; + + const uint16_t a = *(const uint16_t *)x[i].scales; + aux16[0] = a & 0x0f0f; + aux16[1] = (a >> 4) & 0x0f0f; + + vector signed char vscales = (vector signed char)vec_xl_len(scales, 8); + vector signed char qxhs0 = (vector signed char)vec_xl_len(x[i].hmask, 8); + qxhs0 = vec_or(qxhs0, vec_sr(vec_sld(qxhs0, qxhs0, 8), (vector unsigned char)v1)); + + vscales = vec_sub(vscales, off); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs); + vector signed char qxs00 = vec_and(qxs0, lowMask); + vector signed char qxs01 = vec_and(vec_sr(qxs0, v2), lowMask); + vector signed char qxs10 = vec_and(vec_sr(qxs0, v4), lowMask); + vector signed char qxs11 = vec_and(vec_sr(qxs0, v6), lowMask); + + //the 3rd bit + vector signed char qxh00 = vec_sl(vec_andc(v1, qxhs0), v2); + vector signed char qxh01 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v2)), v2); + vector signed char qxh02 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v4)), v2); + vector signed char qxh03 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v6)), v2); + qxhs0 = vec_sr(qxhs0, v4); + + vector signed char q3x00 = vec_sub(qxs00, qxh00); + vector signed char q3x01 = vec_sub(qxs01, qxh01); + vector signed char q3x10 = vec_sub(qxs10, qxh02); + vector signed char q3x11 = vec_sub(qxs11, qxh03); + + vector signed char q8y00 = vec_xl( 0, y[i].qs); + vector signed char q8y01 = vec_xl( 16, y[i].qs); + vector signed char q8y10 = vec_xl( 32, y[i].qs); + vector signed char q8y11 = vec_xl( 48, y[i].qs); + + vector signed short vscales_h = vec_unpackh(vscales); + vector signed short vs0 = vec_splat(vscales_h, 0); + vector signed short vs1 = vec_splat(vscales_h, 1); + vector signed short vs2 = vec_splat(vscales_h, 2); + vector signed short vs3 = vec_splat(vscales_h, 3); + + vector signed short qv00 = vec_add(vec_mule(q3x00, q8y00), vec_mulo(q3x00, q8y00)); + vector signed short qv10 = vec_add(vec_mule(q3x10, q8y10), vec_mulo(q3x10, q8y10)); + vector signed short qv01 = vec_add(vec_mule(q3x01, q8y01), vec_mulo(q3x01, q8y01)); + vector signed short qv11 = vec_add(vec_mule(q3x11, q8y11), vec_mulo(q3x11, q8y11)); + + vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); + vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1)); + vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2)); + vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else int8_t aux8[QK_K]; @@ -6559,6 +7305,142 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin)); + vector float vdmin = vec_mul(vxmin, vyd); + + vector signed short q8ysums0 = vec_xl( 0, y[i].bsums); + vector signed short q8ysums1 = vec_xl(16, y[i].bsums); + + memcpy(utmp, x[i].scales, 12); + + utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4); + const uint32_t uaux = utmp[1] & kmask1; + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[2] = uaux; + utmp[0] &= kmask1; + + vector signed char utmps = (vector signed char)vec_xl( 0, utmp); + vector signed short vscales = vec_unpackh(utmps); + vector signed short q4xmins = vec_unpackl(utmps); + vector signed short q4xmins0 = vec_mergeh(q4xmins, q4xmins); + vector signed short q4xmins1 = vec_mergel(q4xmins, q4xmins); + + vector signed int prod0 = vec_mule(q4xmins0, q8ysums0); + vector signed int prod1 = vec_mule(q4xmins1, q8ysums1); + vector signed int prod2 = vec_mulo(q4xmins0, q8ysums0); + vector signed int prod3 = vec_mulo(q4xmins1, q8ysums1); + + vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0); + vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1); + vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2); + vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + const uint8_t * restrict q4 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/64; j+=2) { + __builtin_prefetch(q4, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, q4); + vector signed char qxs1 = (vector signed char)vec_xl(16, q4); + vector signed char qxs2 = (vector signed char)vec_xl(32, q4); + vector signed char qxs3 = (vector signed char)vec_xl(48, q4); + q4 += 64; + + vector signed char q4x00 = vec_and(qxs0, lowMask); + vector signed char q4x01 = vec_sr(qxs0, v4); + vector signed char q4x10 = vec_and(qxs1, lowMask); + vector signed char q4x11 = vec_sr(qxs1, v4); + vector signed char q4x20 = vec_and(qxs2, lowMask); + vector signed char q4x21 = vec_sr(qxs2, v4); + vector signed char q4x30 = vec_and(qxs3, lowMask); + vector signed char q4x31 = vec_sr(qxs3, v4); + + vector signed char q8y00 = vec_xl( 0, q8); + vector signed char q8y10 = vec_xl( 16, q8); + vector signed char q8y01 = vec_xl( 32, q8); + vector signed char q8y11 = vec_xl( 48, q8); + vector signed char q8y20 = vec_xl( 64, q8); + vector signed char q8y30 = vec_xl( 80, q8); + vector signed char q8y21 = vec_xl( 96, q8); + vector signed char q8y31 = vec_xl(112, q8); + q8 += 128; + + vector signed short qv00 = vec_add(vec_mule(q4x00, q8y00), vec_mulo(q4x00, q8y00)); + vector signed short qv01 = vec_add(vec_mule(q4x01, q8y01), vec_mulo(q4x01, q8y01)); + vector signed short qv10 = vec_add(vec_mule(q4x10, q8y10), vec_mulo(q4x10, q8y10)); + vector signed short qv11 = vec_add(vec_mule(q4x11, q8y11), vec_mulo(q4x11, q8y11)); + vector signed short qv20 = vec_add(vec_mule(q4x20, q8y20), vec_mulo(q4x20, q8y20)); + vector signed short qv21 = vec_add(vec_mule(q4x21, q8y21), vec_mulo(q4x21, q8y21)); + vector signed short qv30 = vec_add(vec_mule(q4x30, q8y30), vec_mulo(q4x30, q8y30)); + vector signed short qv31 = vec_add(vec_mule(q4x31, q8y31), vec_mulo(q4x31, q8y31)); + + vector signed short vs0 = vec_splat(vscales, 0); + vector signed short vs1 = vec_splat(vscales, 1); + vector signed short vs2 = vec_splat(vscales, 2); + vector signed short vs3 = vec_splat(vscales, 3); + vscales = vec_sld(vscales, vscales, 8); + + qv00 = vec_add(qv00, qv10); + qv10 = vec_add(qv01, qv11); + qv20 = vec_add(qv20, qv30); + qv30 = vec_add(qv21, qv31); + + vsumi0 = vec_add(vec_mule(qv00, vs0), vsumi0); + vsumi1 = vec_add(vec_mulo(qv00, vs0), vsumi1); + vsumi2 = vec_add(vec_mule(qv10, vs1), vsumi2); + vsumi3 = vec_add(vec_mulo(qv10, vs1), vsumi3); + vsumi4 = vec_add(vec_mule(qv20, vs2), vsumi4); + vsumi5 = vec_add(vec_mulo(qv20, vs2), vsumi5); + vsumi6 = vec_add(vec_mule(qv30, vs3), vsumi6); + vsumi7 = vec_add(vec_mulo(qv30, vs3), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); + #else @@ -6825,6 +7707,87 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + +#pragma GCC unroll 2 + for (int i = 0; i < nb; ++i) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d[1])); + vector float vyd = vec_splats(y[i].d); + vector float vd= vec_mul(vxd, vyd); + + uint16_t s16[2]; + const uint8_t * scales = (const uint8_t *)s16; + + const uint16_t * restrict b = (const uint16_t *)x[i].scales; + s16[0] = b[0] & 0x0f0f; + s16[1] = (b[0] >> 4) & 0x0f0f; + + vector signed char utmps = (vector signed char)vec_xl_len(scales, 4); + vector signed short vscales = (vector signed short)vec_unpackh(utmps); + vector signed short q4xmins0 = vec_mergeh(vscales, vscales); + q4xmins0 = vec_sld(q4xmins0, q4xmins0, 8); + + vector signed short q8ysums0 = vec_xl_len((const int16_t *)(y[i].bsums), 8); + + vector signed int prod0 = vec_mule(q4xmins0, q8ysums0); + vector signed int prod1 = vec_mulo(q4xmins0, q8ysums0); + + vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vd, vsumf0); + vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vd, vsumf1); + + vd = vec_mul(vyd, vec_splats(GGML_FP16_TO_FP32(x[i].d[0]))); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs); + vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].qs); + vector signed char q4x00 = vec_and(qxs0, lowMask); + vector signed char q4x01 = vec_sr(qxs0, v4); + vector signed char q4x10 = vec_and(qxs1, lowMask); + vector signed char q4x11 = vec_sr(qxs1, v4); + + vector signed char q8y00 = vec_xl( 0, y[i].qs); + vector signed char q8y10 = vec_xl(16, y[i].qs); + vector signed char q8y01 = vec_xl(32, y[i].qs); + vector signed char q8y11 = vec_xl(48, y[i].qs); + + vector signed short qv00 = vec_add(vec_mule(q4x00, q8y00), vec_mulo(q4x00, q8y00)); + vector signed short qv01 = vec_add(vec_mule(q4x01, q8y01), vec_mulo(q4x01, q8y01)); + vector signed short qv10 = vec_add(vec_mule(q4x10, q8y10), vec_mulo(q4x10, q8y10)); + vector signed short qv11 = vec_add(vec_mule(q4x11, q8y11), vec_mulo(q4x11, q8y11)); + + vector signed short vs0 = vec_splat(vscales, 0); + vector signed short vs1 = vec_splat(vscales, 1); + + vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); + vector signed int vsumi1 = vec_add(vec_mule(qv10, vs0), vec_mulo(qv10, vs0)); + vector signed int vsumi2 = vec_add(vec_mule(qv01, vs1), vec_mulo(qv01, vs1)); + vector signed int vsumi3 = vec_add(vec_mule(qv11, vs1), vec_mulo(qv11, vs1)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); + #else uint8_t aux8[QK_K]; @@ -7226,6 +8189,130 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf+sums; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v1 = vec_splats((unsigned char)0x1); + const vector unsigned char v2 = vec_splats((unsigned char)0x2); + const vector unsigned char v3 = vec_splats((unsigned char)0x3); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin)); + vector float vdmin = vec_mul(vxmin, vyd); + + memcpy(utmp, x[i].scales, 12); + + utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4); + const uint32_t uaux = utmp[1] & kmask1; + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[2] = uaux; + utmp[0] &= kmask1; + + vector signed short q8ysums0 = vec_xl( 0, y[i].bsums); + vector signed short q8ysums1 = vec_xl(16, y[i].bsums); + + vector signed char utmps = (vector signed char)vec_xl( 0, utmp); + vector signed short vscales = vec_unpackh(utmps); + + vector signed short q5xmins = vec_unpackl(utmps); + vector signed short q5xmins0 = vec_mergeh(q5xmins, q5xmins); + vector signed short q5xmins1 = vec_mergel(q5xmins, q5xmins); + + vector signed int prod0 = vec_mule(q5xmins0, q8ysums0); + vector signed int prod1 = vec_mule(q5xmins1, q8ysums1); + vector signed int prod2 = vec_mulo(q5xmins0, q8ysums0); + vector signed int prod3 = vec_mulo(q5xmins1, q8ysums1); + + vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0); + vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1); + vsumf2 = vec_nmsub(vec_ctf(prod2, 0), vdmin, vsumf2); + vsumf3 = vec_nmsub(vec_ctf(prod3, 0), vdmin, vsumf3); + + vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].qh); + vector signed char qxhs1 = (vector signed char)vec_xl(16, x[i].qh); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + + const uint8_t * restrict q5 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/64; ++j) { + __builtin_prefetch(q5, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, q5); + vector signed char qxs1 = (vector signed char)vec_xl(16, q5); + q5 += 32; + + vector signed char qxs00 = vec_and(qxs0, lowMask); + vector signed char qxs01 = vec_sr(qxs0, v4); + vector signed char qxs10 = vec_and(qxs1, lowMask); + vector signed char qxs11 = vec_sr(qxs1, v4); + + vector signed char q5h00 = vec_sl(vec_and((vector signed char)v1, qxhs0), v4); + vector signed char q5h01 = vec_sl(vec_and((vector signed char)v2, qxhs0), v3); + vector signed char q5h10 = vec_sl(vec_and((vector signed char)v1, qxhs1), v4); + vector signed char q5h11 = vec_sl(vec_and((vector signed char)v2, qxhs1), v3); + qxhs0 = vec_sr(qxhs0, v2); + qxhs1 = vec_sr(qxhs1, v2); + + vector signed char q5x00 = vec_or(q5h00, qxs00); + vector signed char q5x01 = vec_or(q5h01, qxs01); + vector signed char q5x10 = vec_or(q5h10, qxs10); + vector signed char q5x11 = vec_or(q5h11, qxs11); + + vector signed char q8y00 = vec_xl( 0, q8); + vector signed char q8y10 = vec_xl(16, q8); + vector signed char q8y01 = vec_xl(32, q8); + vector signed char q8y11 = vec_xl(48, q8); + q8 += 64; + + vector signed short qv00 = vec_add(vec_mule(q5x00, q8y00), vec_mulo(q5x00, q8y00)); + vector signed short qv01 = vec_add(vec_mule(q5x01, q8y01), vec_mulo(q5x01, q8y01)); + vector signed short qv10 = vec_add(vec_mule(q5x10, q8y10), vec_mulo(q5x10, q8y10)); + vector signed short qv11 = vec_add(vec_mule(q5x11, q8y11), vec_mulo(q5x11, q8y11)); + + vector signed short vs0 = vec_splat(vscales, 0); + vector signed short vs1 = vec_splat(vscales, 1); + vscales = vec_sld(vscales, vscales, 12); + + qv00 = vec_add(qv00, qv10); + qv01 = vec_add(qv01, qv11); + + vsumi0 = vec_add(vec_mule(qv00, vs0), vsumi0); + vsumi1 = vec_add(vec_mulo(qv00, vs0), vsumi1); + vsumi2 = vec_add(vec_mule(qv01, vs1), vsumi2); + vsumi3 = vec_add(vec_mulo(qv01, vs1), vsumi3); + } + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); + #else const uint8_t * scales = (const uint8_t*)&utmp[0]; @@ -7523,6 +8610,83 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v1 = vec_splats((unsigned char)0x1); + const vector unsigned char v2 = vec_splats((unsigned char)0x2); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + +#pragma GCC unroll 2 + for (int i = 0; i < nb; ++i) { + __builtin_prefetch(x[i].qs, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd= vec_mul(vxd, vyd); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs); + vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].qs); + vector signed char qxs00 = (vector signed char)vec_and(qxs0, lowMask); + vector signed char qxs01 = (vector signed char)vec_sr(qxs0, v4); + vector signed char qxs10 = (vector signed char)vec_and(qxs1, lowMask); + vector signed char qxs11 = (vector signed char)vec_sr(qxs1, v4); + + vector signed char qxhs = (vector signed char)vec_xl_len(x[i].qh, 8); + vector signed char qxhs0 = vec_or(qxhs, vec_sr(vec_sld(qxhs, qxhs, 8), v1)); + vector signed char qxhs1 = vec_sr(qxhs0, v2); + vector signed char qxh00 = vec_sl(vec_andc((vector signed char)v1, qxhs0), v4); + vector signed char qxh10 = vec_sl(vec_andc((vector signed char)v1, qxhs1), v4); + vector signed char qxh01 = vec_sl(vec_andc((vector signed char)v1, vec_sr(qxhs0, v4)), v4); + vector signed char qxh11 = vec_sl(vec_andc((vector signed char)v1, vec_sr(qxhs1, v4)), v4); + + vector signed char q5x00 = vec_sub(qxs00, qxh00); + vector signed char q5x10 = vec_sub(qxs10, qxh10); + vector signed char q5x01 = vec_sub(qxs01, qxh01); + vector signed char q5x11 = vec_sub(qxs11, qxh11); + + vector signed char q8y00 = vec_xl( 0, y[i].qs); + vector signed char q8y10 = vec_xl(16, y[i].qs); + vector signed char q8y01 = vec_xl(32, y[i].qs); + vector signed char q8y11 = vec_xl(48, y[i].qs); + + vector signed short qv00 = vec_add(vec_mule(q5x00, q8y00), vec_mulo(q5x00, q8y00)); + vector signed short qv01 = vec_add(vec_mule(q5x01, q8y01), vec_mulo(q5x01, q8y01)); + vector signed short qv10 = vec_add(vec_mule(q5x10, q8y10), vec_mulo(q5x10, q8y10)); + vector signed short qv11 = vec_add(vec_mule(q5x11, q8y11), vec_mulo(q5x11, q8y11)); + + vector signed short vs = (vector signed short)vec_unpackh(vec_xl_len(x[i].scales, 4)); + vector signed short vs0 = vec_splat(vs, 0); + vector signed short vs1 = vec_splat(vs, 1); + vector signed short vs2 = vec_splat(vs, 2); + vector signed short vs3 = vec_splat(vs, 3); + + vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); + vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1)); + vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2)); + vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); + #else int8_t aux8[QK_K]; @@ -7953,6 +9117,151 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v2 = vec_splats((unsigned char)0x2); + const vector unsigned char v3 = vec_splats((unsigned char)0x3); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + const vector unsigned char v6 = vec_splats((unsigned char)0x6); + const vector signed char off = vec_splats((signed char)0x20); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + const uint8_t * restrict q6 = x[i].ql; + const uint8_t * restrict qh = x[i].qh; + const int8_t * restrict qs = x[i].scales; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/128; ++j) { + __builtin_prefetch(q6, 0, 0); + __builtin_prefetch(qh, 0, 0); + __builtin_prefetch(q8, 0, 0); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, q6); + vector signed char qxs1 = (vector signed char)vec_xl(16, q6); + vector signed char qxs2 = (vector signed char)vec_xl(32, q6); + vector signed char qxs3 = (vector signed char)vec_xl(48, q6); + q6 += 64; + + vector signed char qxs00 = vec_and(qxs0, lowMask); + vector signed char qxs01 = vec_sr(qxs0, v4); + vector signed char qxs10 = vec_and(qxs1, lowMask); + vector signed char qxs11 = vec_sr(qxs1, v4); + vector signed char qxs20 = vec_and(qxs2, lowMask); + vector signed char qxs21 = vec_sr(qxs2, v4); + vector signed char qxs30 = vec_and(qxs3, lowMask); + vector signed char qxs31 = vec_sr(qxs3, v4); + + vector signed char qxhs0 = (vector signed char)vec_xl( 0, qh); + vector signed char qxhs1 = (vector signed char)vec_xl(16, qh); + qh += 32; + + vector signed char qxh00 = vec_sl(vec_and((vector signed char)v3, qxhs0), v4); + vector signed char qxh01 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v4)), v4); + vector signed char qxh10 = vec_sl(vec_and((vector signed char)v3, qxhs1), v4); + vector signed char qxh11 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v4)), v4); + vector signed char qxh20 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v2)), v4); + vector signed char qxh21 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v6)), v4); + vector signed char qxh30 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v2)), v4); + vector signed char qxh31 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs1, v6)), v4); + + vector signed char q6x00 = vec_sub(vec_or(qxh00, qxs00), off); + vector signed char q6x01 = vec_sub(vec_or(qxh01, qxs01), off); + vector signed char q6x10 = vec_sub(vec_or(qxh10, qxs10), off); + vector signed char q6x11 = vec_sub(vec_or(qxh11, qxs11), off); + vector signed char q6x20 = vec_sub(vec_or(qxh20, qxs20), off); + vector signed char q6x21 = vec_sub(vec_or(qxh21, qxs21), off); + vector signed char q6x30 = vec_sub(vec_or(qxh30, qxs30), off); + vector signed char q6x31 = vec_sub(vec_or(qxh31, qxs31), off); + + vector signed char q8y00 = vec_xl( 0, q8); + vector signed char q8y10 = vec_xl( 16, q8); + vector signed char q8y20 = vec_xl( 32, q8); + vector signed char q8y30 = vec_xl( 48, q8); + vector signed char q8y01 = vec_xl( 64, q8); + vector signed char q8y11 = vec_xl( 80, q8); + vector signed char q8y21 = vec_xl( 96, q8); + vector signed char q8y31 = vec_xl(112, q8); + q8 += 128; + + vector signed short qv00 = vec_add(vec_mule(q6x00, q8y00), vec_mulo(q6x00, q8y00)); + vector signed short qv10 = vec_add(vec_mule(q6x10, q8y10), vec_mulo(q6x10, q8y10)); + vector signed short qv20 = vec_add(vec_mule(q6x20, q8y20), vec_mulo(q6x20, q8y20)); + vector signed short qv30 = vec_add(vec_mule(q6x30, q8y30), vec_mulo(q6x30, q8y30)); + vector signed short qv01 = vec_add(vec_mule(q6x01, q8y01), vec_mulo(q6x01, q8y01)); + vector signed short qv11 = vec_add(vec_mule(q6x11, q8y11), vec_mulo(q6x11, q8y11)); + vector signed short qv21 = vec_add(vec_mule(q6x21, q8y21), vec_mulo(q6x21, q8y21)); + vector signed short qv31 = vec_add(vec_mule(q6x31, q8y31), vec_mulo(q6x31, q8y31)); + + vector signed short vscales = vec_unpackh(vec_xl_len(qs, 8)); + qs += 8; + + vector signed short vs0 = vec_splat(vscales, 0); + vector signed short vs1 = vec_splat(vscales, 1); + vector signed short vs2 = vec_splat(vscales, 2); + vector signed short vs3 = vec_splat(vscales, 3); + vector signed short vs4 = vec_splat(vscales, 4); + vector signed short vs5 = vec_splat(vscales, 5); + vector signed short vs6 = vec_splat(vscales, 6); + vector signed short vs7 = vec_splat(vscales, 7); + + vsumi0 = vec_add(vec_mule(qv00, vs0), vsumi0); + vsumi1 = vec_add(vec_mulo(qv00, vs0), vsumi1); + vsumi2 = vec_add(vec_mule(qv01, vs4), vsumi2); + vsumi3 = vec_add(vec_mulo(qv01, vs4), vsumi3); + vsumi4 = vec_add(vec_mule(qv10, vs1), vsumi4); + vsumi5 = vec_add(vec_mulo(qv10, vs1), vsumi5); + vsumi6 = vec_add(vec_mule(qv11, vs5), vsumi6); + vsumi7 = vec_add(vec_mulo(qv11, vs5), vsumi7); + + vsumi0 = vec_add(vec_mule(qv20, vs2), vsumi0); + vsumi1 = vec_add(vec_mulo(qv20, vs2), vsumi1); + vsumi2 = vec_add(vec_mule(qv21, vs6), vsumi2); + vsumi3 = vec_add(vec_mulo(qv21, vs6), vsumi3); + vsumi4 = vec_add(vec_mule(qv30, vs3), vsumi4); + vsumi5 = vec_add(vec_mulo(qv30, vs3), vsumi5); + vsumi6 = vec_add(vec_mule(qv31, vs7), vsumi6); + vsumi7 = vec_add(vec_mulo(qv31, vs7), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); + #else int8_t aux8[QK_K]; @@ -8259,6 +9568,85 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v2 = vec_splats((unsigned char)0x2); + const vector unsigned char v3 = vec_splats((unsigned char)0x3); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + const vector unsigned char v6 = vec_splats((unsigned char)0x6); + const vector signed char off = vec_splats((signed char)0x20); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + +#pragma GCC unroll 2 + for (int i = 0; i < nb; ++i) { + __builtin_prefetch(x[i].ql, 0, 1); + __builtin_prefetch(x[i].qh, 0, 1); + __builtin_prefetch(y[i].qs, 0, 1); + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd= vec_mul(vxd, vyd); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].ql); + vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].ql); + vector signed char qxs00 = vec_and(qxs0, lowMask); + vector signed char qxs01 = vec_sr(qxs0, v4); + vector signed char qxs10 = vec_and(qxs1, lowMask); + vector signed char qxs11 = vec_sr(qxs1, v4); + + vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].qh); + + vector signed char qxh00 = vec_sl(vec_and((vector signed char)v3, qxhs0), v4); + vector signed char qxh01 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v4)), v4); + vector signed char qxh10 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v2)), v4); + vector signed char qxh11 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v6)), v4); + + vector signed char q6x00 = vec_sub(vec_or(qxh00, qxs00), off); + vector signed char q6x01 = vec_sub(vec_or(qxh01, qxs01), off); + vector signed char q6x10 = vec_sub(vec_or(qxh10, qxs10), off); + vector signed char q6x11 = vec_sub(vec_or(qxh11, qxs11), off); + + vector signed char q8y00 = vec_xl( 0, y[i].qs); + vector signed char q8y10 = vec_xl(16, y[i].qs); + vector signed char q8y01 = vec_xl(32, y[i].qs); + vector signed char q8y11 = vec_xl(48, y[i].qs); + + vector signed short qv00 = vec_add(vec_mule(q6x00, q8y00), vec_mulo(q6x00, q8y00)); + vector signed short qv10 = vec_add(vec_mule(q6x10, q8y10), vec_mulo(q6x10, q8y10)); + vector signed short qv01 = vec_add(vec_mule(q6x01, q8y01), vec_mulo(q6x01, q8y01)); + vector signed short qv11 = vec_add(vec_mule(q6x11, q8y11), vec_mulo(q6x11, q8y11)); + + vector signed short vs = (vector signed short)vec_unpackh(vec_xl_len(x[i].scales, 4)); + vector signed short vs0 = vec_splat(vs, 0); + vector signed short vs1 = vec_splat(vs, 1); + vector signed short vs2 = vec_splat(vs, 2); + vector signed short vs3 = vec_splat(vs, 3); + + vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); + vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1)); + vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2)); + vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); + #else int8_t aux8[QK_K]; @@ -8300,7 +9688,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r #endif -#if defined (__AVX2__) || defined (__ARM_NEON) +#if defined (__AVX2__) || defined (__ARM_NEON) || defined (__POWER9_VECTOR__) static const int8_t keven_signs_q2xs[1024] = { 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1, @@ -8433,6 +9821,103 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void *s = 0.125f * hsum_float_8(accumf); +#elif defined(__POWER9_VECTOR__) + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + const uint16_t * restrict q2 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/32; j += 2) { + __builtin_prefetch(q2, 0, 1); + __builtin_prefetch(q8, 0, 1); + + uint32_t aux32[4]; + const uint8_t * aux8 = (const uint8_t *)aux32; + + memcpy(aux32, q2, 4*sizeof(uint32_t)); + q2 += 8; + + vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1])}; + vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3])}; + vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9])}; + vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11])}; + + vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((aux32[1] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 7) & 127))}; + vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127))}; + vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((aux32[3] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 7) & 127))}; + vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127))}; + + vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0); + vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1); + vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2); + vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3); + + vector signed char q8y0 = vec_xl( 0, q8); + vector signed char q8y1 = vec_xl(16, q8); + vector signed char q8y2 = vec_xl(32, q8); + vector signed char q8y3 = vec_xl(48, q8); + q8 += 64; + + vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1)); + vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2)); + vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3)); + + const uint16_t ls0 = aux32[1] >> 28; + const uint16_t ls1 = aux32[3] >> 28; + + vector signed short vscales01 = vec_splats((int16_t)(2*ls0+1)); + vector signed short vscales23 = vec_splats((int16_t)(2*ls1+1)); + + vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0); + vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1); + vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2); + vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3); + vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4); + vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5); + vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6); + vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = 0.125f * vec_extract(vsumf0, 0); #else uint32_t aux32[2]; @@ -8708,6 +10193,104 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * *s = 0.125f * hsum_float_8(accumf); #endif +#elif defined(__POWER9_VECTOR__) + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + const uint16_t * restrict q2 = x[i].qs; + const uint8_t * restrict sc = x[i].scales; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/64; ++j) { + __builtin_prefetch(q2, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector signed long long aux64x2_0 = {*(const int64_t *)(iq2xs_grid + (q2[0] & 511)), *(const int64_t *)(iq2xs_grid + (q2[1] & 511))}; + vector signed long long aux64x2_1 = {*(const int64_t *)(iq2xs_grid + (q2[2] & 511)), *(const int64_t *)(iq2xs_grid + (q2[3] & 511))}; + vector signed long long aux64x2_2 = {*(const int64_t *)(iq2xs_grid + (q2[4] & 511)), *(const int64_t *)(iq2xs_grid + (q2[5] & 511))}; + vector signed long long aux64x2_3 = {*(const int64_t *)(iq2xs_grid + (q2[6] & 511)), *(const int64_t *)(iq2xs_grid + (q2[7] & 511))}; + + vector signed long long vsigns0 = {*(const int64_t *)(signs64 + ((q2[0] >> 9))), *(const int64_t *)(signs64 + ((q2[1] >> 9)))}; + vector signed long long vsigns1 = {*(const int64_t *)(signs64 + ((q2[2] >> 9))), *(const int64_t *)(signs64 + ((q2[3] >> 9)))}; + vector signed long long vsigns2 = {*(const int64_t *)(signs64 + ((q2[4] >> 9))), *(const int64_t *)(signs64 + ((q2[5] >> 9)))}; + vector signed long long vsigns3 = {*(const int64_t *)(signs64 + ((q2[6] >> 9))), *(const int64_t *)(signs64 + ((q2[7] >> 9)))}; + q2 += 8; + + vector signed char q2x0 = (vector signed char)vec_mul((vector signed char)vsigns0, (vector signed char)aux64x2_0); + vector signed char q2x1 = (vector signed char)vec_mul((vector signed char)vsigns1, (vector signed char)aux64x2_1); + vector signed char q2x2 = (vector signed char)vec_mul((vector signed char)vsigns2, (vector signed char)aux64x2_2); + vector signed char q2x3 = (vector signed char)vec_mul((vector signed char)vsigns3, (vector signed char)aux64x2_3); + + vector signed char q8y0 = vec_xl( 0, q8); + vector signed char q8y1 = vec_xl(16, q8); + vector signed char q8y2 = vec_xl(32, q8); + vector signed char q8y3 = vec_xl(48, q8); + q8 += 64; + + vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1)); + vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2)); + vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3)); + + const uint16_t ls0 = (uint16_t)(sc[0] & 0xf); + const uint16_t ls1 = (uint16_t)(sc[0] >> 4); + const uint16_t ls2 = (uint16_t)(sc[1] & 0xf); + const uint16_t ls3 = (uint16_t)(sc[1] >> 4); + sc += 2; + + vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1)); + vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1)); + vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1)); + vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1)); + + vsumi0 = vec_add(vec_mule(qv0, vscales0), vsumi0); + vsumi1 = vec_add(vec_mule(qv1, vscales1), vsumi1); + vsumi2 = vec_add(vec_mule(qv2, vscales2), vsumi2); + vsumi3 = vec_add(vec_mule(qv3, vscales3), vsumi3); + vsumi4 = vec_add(vec_mulo(qv0, vscales0), vsumi4); + vsumi5 = vec_add(vec_mulo(qv1, vscales1), vsumi5); + vsumi6 = vec_add(vec_mulo(qv2, vscales2), vsumi6); + vsumi7 = vec_add(vec_mulo(qv3, vscales3), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = 0.125f * vec_extract(vsumf0, 0); #else float sumf = 0.f; @@ -8908,6 +10491,124 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * *s = 0.125f * hsum_float_8(accumf); +#elif defined(__POWER9_VECTOR__) + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 + }; + + static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,}; + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + const vector unsigned char mask0 = vec_xl( 0, k_mask1); + const vector unsigned char mask1 = vec_xl(16, k_mask1); + const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + const uint8_t * restrict q2 = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8); + const uint8_t * restrict sc = x[i].scales; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/32; j += 2) { + __builtin_prefetch(q2, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector signed long long aux64x2_0 = {*(const int64_t *)(iq2s_grid + (q2[0] | ((qh[0] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[1] | ((qh[0] << 6) & 0x300)))}; + vector signed long long aux64x2_1 = {*(const int64_t *)(iq2s_grid + (q2[2] | ((qh[0] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[3] | ((qh[0] << 2) & 0x300)))}; + vector signed long long aux64x2_2 = {*(const int64_t *)(iq2s_grid + (q2[4] | ((qh[1] << 8) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[5] | ((qh[1] << 6) & 0x300)))}; + vector signed long long aux64x2_3 = {*(const int64_t *)(iq2s_grid + (q2[6] | ((qh[1] << 4) & 0x300))), *(const int64_t *)(iq2s_grid + (q2[7] | ((qh[1] << 2) & 0x300)))}; + q2 += 8; + qh += 2; + + vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]); + vector signed char vsigns23 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]); + signs += 4; + + vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0); + vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1); + vector signed char vsigns2 = vec_perm(vsigns23, vsigns23, mask0); + vector signed char vsigns3 = vec_perm(vsigns23, vsigns23, mask1); + + vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2); + vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2); + vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2); + vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2); + + vector signed char q2x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux64x2_0), vsigns0); + vector signed char q2x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux64x2_1), vsigns1); + vector signed char q2x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux64x2_2), vsigns2); + vector signed char q2x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux64x2_3), vsigns3); + + vector signed char q8y0 = vec_xl( 0, q8); + vector signed char q8y1 = vec_xl(16, q8); + vector signed char q8y2 = vec_xl(32, q8); + vector signed char q8y3 = vec_xl(48, q8); + q8 += 64; + + vector signed short qv0 = vec_add(vec_mule(q2x0, q8y0), vec_mulo(q2x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q2x1, q8y1), vec_mulo(q2x1, q8y1)); + vector signed short qv2 = vec_add(vec_mule(q2x2, q8y2), vec_mulo(q2x2, q8y2)); + vector signed short qv3 = vec_add(vec_mule(q2x3, q8y3), vec_mulo(q2x3, q8y3)); + + const uint16_t ls0 = (uint16_t)(sc[0] & 0xf); + const uint16_t ls1 = (uint16_t)(sc[0] >> 4); + const uint16_t ls2 = (uint16_t)(sc[1] & 0xf); + const uint16_t ls3 = (uint16_t)(sc[1] >> 4); + sc += 2; + + vector signed short vscales0 = vec_splats((int16_t)(2*ls0+1)); + vector signed short vscales1 = vec_splats((int16_t)(2*ls1+1)); + vector signed short vscales2 = vec_splats((int16_t)(2*ls2+1)); + vector signed short vscales3 = vec_splats((int16_t)(2*ls3+1)); + + vsumi0 = vec_add(vec_mule(qv0, vscales0), vsumi0); + vsumi1 = vec_add(vec_mule(qv1, vscales1), vsumi1); + vsumi2 = vec_add(vec_mule(qv2, vscales2), vsumi2); + vsumi3 = vec_add(vec_mule(qv3, vscales3), vsumi3); + vsumi4 = vec_add(vec_mulo(qv0, vscales0), vsumi4); + vsumi5 = vec_add(vec_mulo(qv1, vscales1), vsumi5); + vsumi6 = vec_add(vec_mulo(qv2, vscales2), vsumi6); + vsumi7 = vec_add(vec_mulo(qv3, vscales3), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = 0.125f * vec_extract(vsumf0, 0); #else float sumf = 0; @@ -9052,6 +10753,101 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void *s = 0.25f * hsum_float_8(accumf); +#elif defined(__POWER9_VECTOR__) + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + const uint8_t * restrict q3 = x[i].qs; + const uint32_t * restrict signs = (const uint32_t *)(x[i].qs + QK_K/4); + const int8_t * restrict q8 = y[i].qs; + +#pragma GCC unroll 1 + for (int j = 0; j < QK_K/32; j += 2) { + __builtin_prefetch(q3, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector unsigned int aux32x4_0 = {iq3xxs_grid[q3[ 0]], iq3xxs_grid[q3[ 1]], iq3xxs_grid[q3[ 2]], iq3xxs_grid[q3[ 3]]}; + vector unsigned int aux32x4_1 = {iq3xxs_grid[q3[ 4]], iq3xxs_grid[q3[ 5]], iq3xxs_grid[q3[ 6]], iq3xxs_grid[q3[ 7]]}; + vector unsigned int aux32x4_2 = {iq3xxs_grid[q3[ 8]], iq3xxs_grid[q3[ 9]], iq3xxs_grid[q3[10]], iq3xxs_grid[q3[11]]}; + vector unsigned int aux32x4_3 = {iq3xxs_grid[q3[12]], iq3xxs_grid[q3[13]], iq3xxs_grid[q3[14]], iq3xxs_grid[q3[15]]}; + q3 += 16; + + vector unsigned long long aux64x2_0 = {(uint64_t)(signs64[(signs[0] >> 0) & 127]), (uint64_t)(signs64[(signs[0] >> 7) & 127])}; + vector unsigned long long aux64x2_1 = {(uint64_t)(signs64[(signs[0] >> 14) & 127]), (uint64_t)(signs64[(signs[0] >> 21) & 127])}; + vector unsigned long long aux64x2_2 = {(uint64_t)(signs64[(signs[1] >> 0) & 127]), (uint64_t)(signs64[(signs[1] >> 7) & 127])}; + vector unsigned long long aux64x2_3 = {(uint64_t)(signs64[(signs[1] >> 14) & 127]), (uint64_t)(signs64[(signs[1] >> 21) & 127])}; + + vector signed char q3x0 = vec_mul((vector signed char)aux64x2_0, (vector signed char)aux32x4_0); + vector signed char q3x1 = vec_mul((vector signed char)aux64x2_1, (vector signed char)aux32x4_1); + vector signed char q3x2 = vec_mul((vector signed char)aux64x2_2, (vector signed char)aux32x4_2); + vector signed char q3x3 = vec_mul((vector signed char)aux64x2_3, (vector signed char)aux32x4_3); + + vector signed char q8y0 = vec_xl( 0, q8); + vector signed char q8y1 = vec_xl(16, q8); + vector signed char q8y2 = vec_xl(32, q8); + vector signed char q8y3 = vec_xl(48, q8); + q8 += 64; + + vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1)); + vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2)); + vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3)); + + const uint16_t ls0 = (uint16_t)(signs[0] >> 28); + const uint16_t ls1 = (uint16_t)(signs[1] >> 28); + signs += 2; + + vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1)); + vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1)); + + vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0); + vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1); + vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2); + vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3); + vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4); + vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5); + vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6); + vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = 0.25f * vec_extract(vsumf0, 0); #else uint32_t aux32; @@ -9279,6 +11075,124 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * *s = hsum_float_8(accumf); +#elif defined(__POWER9_VECTOR__) + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 + }; + + static const uint8_t k_mask2[16] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,}; + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + const vector unsigned char mask0 = vec_xl( 0, k_mask1); + const vector unsigned char mask1 = vec_xl(16, k_mask1); + const vector signed char mask2 = (vector signed char)vec_xl( 0, k_mask2); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + const uint8_t * restrict q3 = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint16_t * restrict signs = (const uint16_t *)(x[i].signs); + const uint8_t * restrict sc = x[i].scales; + const int8_t * restrict q8 = y[i].qs; + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + for (int j = 0; j < QK_K/32; j += 2) { + __builtin_prefetch(q3, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector unsigned int aux32x4_0 = {iq3s_grid[q3[ 0] | ((qh[0] << 8) & 256)], iq3s_grid[q3[ 1] | ((qh[0] << 7) & 256)], + iq3s_grid[q3[ 2] | ((qh[0] << 6) & 256)], iq3s_grid[q3[ 3] | ((qh[0] << 5) & 256)]}; + vector unsigned int aux32x4_1 = {iq3s_grid[q3[ 4] | ((qh[0] << 4) & 256)], iq3s_grid[q3[ 5] | ((qh[0] << 3) & 256)], + iq3s_grid[q3[ 6] | ((qh[0] << 2) & 256)], iq3s_grid[q3[ 7] | ((qh[0] << 1) & 256)]}; + vector unsigned int aux32x4_2 = {iq3s_grid[q3[ 8] | ((qh[1] << 8) & 256)], iq3s_grid[q3[ 9] | ((qh[1] << 7) & 256)], + iq3s_grid[q3[10] | ((qh[1] << 6) & 256)], iq3s_grid[q3[11] | ((qh[1] << 5) & 256)]}; + vector unsigned int aux32x4_3 = {iq3s_grid[q3[12] | ((qh[1] << 4) & 256)], iq3s_grid[q3[13] | ((qh[1] << 3) & 256)], + iq3s_grid[q3[14] | ((qh[1] << 2) & 256)], iq3s_grid[q3[15] | ((qh[1] << 1) & 256)]}; + q3 += 16; + qh += 2; + + vector signed char vsigns01 = (vector signed char)vec_splats(*(const uint32_t *)&signs[0]); + vector signed char vsigns02 = (vector signed char)vec_splats(*(const uint32_t *)&signs[2]); + signs += 4; + + vector signed char vsigns0 = vec_perm(vsigns01, vsigns01, mask0); + vector signed char vsigns1 = vec_perm(vsigns01, vsigns01, mask1); + vector signed char vsigns2 = vec_perm(vsigns02, vsigns02, mask0); + vector signed char vsigns3 = vec_perm(vsigns02, vsigns02, mask1); + + vsigns0 = (vector signed char)vec_cmpeq(vec_and(vsigns0, mask2), mask2); + vsigns1 = (vector signed char)vec_cmpeq(vec_and(vsigns1, mask2), mask2); + vsigns2 = (vector signed char)vec_cmpeq(vec_and(vsigns2, mask2), mask2); + vsigns3 = (vector signed char)vec_cmpeq(vec_and(vsigns3, mask2), mask2); + + vector signed char q3x0 = vec_sub(vec_xor(vsigns0, (vector signed char)aux32x4_0), vsigns0); + vector signed char q3x1 = vec_sub(vec_xor(vsigns1, (vector signed char)aux32x4_1), vsigns1); + vector signed char q3x2 = vec_sub(vec_xor(vsigns2, (vector signed char)aux32x4_2), vsigns2); + vector signed char q3x3 = vec_sub(vec_xor(vsigns3, (vector signed char)aux32x4_3), vsigns3); + + vector signed char q8y0 = vec_xl( 0, q8); + vector signed char q8y1 = vec_xl(16, q8); + vector signed char q8y2 = vec_xl(32, q8); + vector signed char q8y3 = vec_xl(48, q8); + q8 += 64; + + vector signed short qv0 = vec_add(vec_mule(q3x0, q8y0), vec_mulo(q3x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q3x1, q8y1), vec_mulo(q3x1, q8y1)); + vector signed short qv2 = vec_add(vec_mule(q3x2, q8y2), vec_mulo(q3x2, q8y2)); + vector signed short qv3 = vec_add(vec_mule(q3x3, q8y3), vec_mulo(q3x3, q8y3)); + + const uint16_t ls0 = (uint16_t)(sc[0] & 0xf); + const uint16_t ls1 = (uint16_t)(sc[0] >> 4); + sc ++; + + vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1)); + vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1)); + + vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0); + vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1); + vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2); + vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3); + vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4); + vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5); + vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6); + vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else float sumf = 0.f; @@ -9433,6 +11347,113 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void *s = hsum_float_8(accum) + IQ1S_DELTA * accum1; +#elif defined(__POWER9_VECTOR__) + const vector unsigned char v0 = vec_splats((unsigned char)0x0); + const vector unsigned short vsign = vec_splats((unsigned short)0x8000); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + for (int i = 0; i < nb; ++i) { + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); + vector float vyd = vec_splats(y[i].d); + vector float vd = vec_mul(vxd, vyd); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + vector signed int vsumi8 = vec_splats((int32_t)0); + + const uint8_t * restrict q1 = x[i].qs; + const uint16_t * restrict qh = x[i].qh; + const int8_t * restrict q8 = y[i].qs; + const int16_t * restrict qs = y[i].bsums; + + for (int j = 0; j < QK_K/32; j += 2) { + __builtin_prefetch(q1, 0, 1); + __builtin_prefetch(qh, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector signed long long aux64x2_0 = {*(const int64_t *)(iq1s_grid + (q1[0] | ((qh[0] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[1] | ((qh[0] << 5) & 0x700)))}; + vector signed long long aux64x2_1 = {*(const int64_t *)(iq1s_grid + (q1[2] | ((qh[0] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[3] | ((qh[0] >> 1) & 0x700)))}; + vector signed long long aux64x2_2 = {*(const int64_t *)(iq1s_grid + (q1[4] | ((qh[1] << 8) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[5] | ((qh[1] << 5) & 0x700)))}; + vector signed long long aux64x2_3 = {*(const int64_t *)(iq1s_grid + (q1[6] | ((qh[1] << 2) & 0x700))), *(const int64_t *)(iq1s_grid + (q1[7] | ((qh[1] >> 1) & 0x700)))}; + q1 += 8; + + vector signed char q1x0 = (vector signed char)aux64x2_0; + vector signed char q1x1 = (vector signed char)aux64x2_1; + vector signed char q1x2 = (vector signed char)aux64x2_2; + vector signed char q1x3 = (vector signed char)aux64x2_3; + + vector signed char q8y0 = vec_xl( 0, q8); + vector signed char q8y1 = vec_xl(16, q8); + vector signed char q8y2 = vec_xl(32, q8); + vector signed char q8y3 = vec_xl(48, q8); + q8 += 64; + + vector signed short qv0 = vec_add(vec_mule(q1x0, q8y0), vec_mulo(q1x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q1x1, q8y1), vec_mulo(q1x1, q8y1)); + vector signed short qv2 = vec_add(vec_mule(q1x2, q8y2), vec_mulo(q1x2, q8y2)); + vector signed short qv3 = vec_add(vec_mule(q1x3, q8y3), vec_mulo(q1x3, q8y3)); + + const uint16_t ls0 = (uint16_t)((qh[0] >> 12) & 7); + const uint16_t ls1 = (uint16_t)((qh[1] >> 12) & 7); + + vector signed short vscales01 = (vector signed short)vec_splats((uint16_t)(2*ls0+1)); + vector signed short vscales23 = (vector signed short)vec_splats((uint16_t)(2*ls1+1)); + vector signed short vscales = vec_sld(vscales23, vscales01, 8); + + vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0); + vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1); + vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2); + vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3); + vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4); + vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5); + vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6); + vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7); + + vector signed short q8ysums = vec_xl_len(qs, 8); + qs += 4; + q8ysums = vec_mergeh(q8ysums, (vector signed short)v0); + + vector signed short qxh = (vector signed short)vec_sld(vec_splats(qh[1]), vec_splats(qh[0]), 8); + qh += 2; + vector bool short vsel = vec_cmpge(qxh, (vector signed short)v0); + + vector signed short q8ysum = vec_sel((vector signed short)vec_xor((vector unsigned short)q8ysums, vsign), q8ysums, vsel); + + vsumi8 = vec_add(vec_mule(q8ysum, vscales), vsumi8); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + + vsumf0 = vec_madd(vec_ctf(vsumi8, 0), vec_mul(vd, vec_splats(IQ1S_DELTA)), vsumf0); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else float sumf = 0; @@ -9789,6 +11810,51 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * *s = hsum_float_8(_mm256_add_ps(accum1, accum2)); +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + + const vector signed char values = vec_xl( 0, kvalues_iq4nl); + +#pragma GCC unroll 4 + for (int ib = 0; ib < nb; ++ib) { + __builtin_prefetch(x[ib].qs, 0, 1); + __builtin_prefetch(y[ib].qs, 0, 1); + + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ib].d)); + vector float vyd = vec_splats(GGML_FP16_TO_FP32(y[ib].d)); + vector float vd = vec_mul(vxd, vyd); + + vector signed char qxs = (vector signed char)vec_xl( 0, x[ib].qs); + vector signed char q4x0 = vec_and(qxs, lowMask); + vector signed char q4x1 = vec_sr(qxs, v4); + + q4x0 = vec_perm(values, values, (vector unsigned char)q4x0); + q4x1 = vec_perm(values, values, (vector unsigned char)q4x1); + + vector signed char q8y0 = vec_xl( 0, y[ib].qs); + vector signed char q8y1 = vec_xl(16, y[ib].qs); + + vector signed short qv0 = vec_add(vec_mule(q4x0, q8y0), vec_mulo(q4x0, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q4x1, q8y1), vec_mulo(q4x1, q8y1)); + + vector signed int vsumi0 = vec_add(vec_unpackh(qv0), vec_unpackl(qv0)); + vector signed int vsumi1 = vec_add(vec_unpackh(qv1), vec_unpackl(qv1)); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + } + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else float sumf = 0; for (int ib = 0; ib < nb; ++ib) { @@ -9900,6 +11966,105 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * *s = hsum_float_8(accum); +#elif defined(__POWER9_VECTOR__) + const vector signed char lowMask = vec_splats((signed char)0xF); + const vector unsigned char v4 = vec_splats((unsigned char)0x4); + + vector float vsumf0 = vec_splats(0.0f); + vector float vsumf1 = vec_splats(0.0f); + vector float vsumf2 = vec_splats(0.0f); + vector float vsumf3 = vec_splats(0.0f); + + const vector signed char values = vec_xl( 0, kvalues_iq4nl); + + for (int ibl = 0; ibl < nb; ++ibl) { + + vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[ibl].d)); + vector float vyd = vec_splats(y[ibl].d); + vector float vd = vec_mul(vxd, vyd); + + vector signed int vsumi0 = vec_splats((int32_t)0); + vector signed int vsumi1 = vec_splats((int32_t)0); + vector signed int vsumi2 = vec_splats((int32_t)0); + vector signed int vsumi3 = vec_splats((int32_t)0); + vector signed int vsumi4 = vec_splats((int32_t)0); + vector signed int vsumi5 = vec_splats((int32_t)0); + vector signed int vsumi6 = vec_splats((int32_t)0); + vector signed int vsumi7 = vec_splats((int32_t)0); + + uint16_t h = x[ibl].scales_h; + + const uint8_t * restrict q4 = x[ibl].qs; + const uint8_t * restrict sc = x[ibl].scales_l; + const int8_t * restrict q8 = y[ibl].qs; + + for (int ib = 0; ib < QK_K/64; ib ++ ) { + __builtin_prefetch(q4, 0, 1); + __builtin_prefetch(q8, 0, 1); + + vector signed char qxs0 = (vector signed char)vec_xl( 0, q4); + vector signed char qxs1 = (vector signed char)vec_xl(16, q4); + q4 += 32; + + vector signed char q4x00 = (vector signed char)vec_and(qxs0, lowMask); + vector signed char q4x01 = (vector signed char)vec_sr(qxs0, v4); + vector signed char q4x10 = (vector signed char)vec_and(qxs1, lowMask); + vector signed char q4x11 = (vector signed char)vec_sr(qxs1, v4); + + q4x00 = vec_perm(values, values, (vector unsigned char)q4x00); + q4x01 = vec_perm(values, values, (vector unsigned char)q4x01); + q4x10 = vec_perm(values, values, (vector unsigned char)q4x10); + q4x11 = vec_perm(values, values, (vector unsigned char)q4x11); + + vector signed char q8y0 = vec_xl( 0, q8); + vector signed char q8y1 = vec_xl(16, q8); + vector signed char q8y2 = vec_xl(32, q8); + vector signed char q8y3 = vec_xl(48, q8); + q8 += 64; + + vector signed short qv0 = vec_add(vec_mule(q4x00, q8y0), vec_mulo(q4x00, q8y0)); + vector signed short qv1 = vec_add(vec_mule(q4x01, q8y1), vec_mulo(q4x01, q8y1)); + vector signed short qv2 = vec_add(vec_mule(q4x10, q8y2), vec_mulo(q4x10, q8y2)); + vector signed short qv3 = vec_add(vec_mule(q4x11, q8y3), vec_mulo(q4x11, q8y3)); + + const uint16_t ls0 = (uint16_t)(((sc[0] & 0xf) | ((h << 4) & 0x30)) - 32); + const uint16_t ls1 = (uint16_t)(((sc[0] >> 4) | ((h << 2) & 0x30)) - 32); + h >>= 4; + sc ++; + + vector signed short vscales01 = vec_splats((int16_t)ls0); + vector signed short vscales23 = vec_splats((int16_t)ls1); + + vsumi0 = vec_add(vec_mule(qv0, vscales01), vsumi0); + vsumi1 = vec_add(vec_mule(qv1, vscales01), vsumi1); + vsumi2 = vec_add(vec_mule(qv2, vscales23), vsumi2); + vsumi3 = vec_add(vec_mule(qv3, vscales23), vsumi3); + vsumi4 = vec_add(vec_mulo(qv0, vscales01), vsumi4); + vsumi5 = vec_add(vec_mulo(qv1, vscales01), vsumi5); + vsumi6 = vec_add(vec_mulo(qv2, vscales23), vsumi6); + vsumi7 = vec_add(vec_mulo(qv3, vscales23), vsumi7); + } + + vsumi0 = vec_add(vsumi0, vsumi4); + vsumi1 = vec_add(vsumi1, vsumi5); + vsumi2 = vec_add(vsumi2, vsumi6); + vsumi3 = vec_add(vsumi3, vsumi7); + + vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); + vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); + vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); + vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); + } + + vsumf0 = vec_add(vsumf0, vsumf2); + vsumf1 = vec_add(vsumf1, vsumf3); + + vsumf0 = vec_add(vsumf0, vsumf1); + + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); + vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); + + *s = vec_extract(vsumf0, 0); #else float sumf = 0; for (int ibl = 0; ibl < nb; ++ibl) { From 182adefcf36fc5f4263082ff032c0796fda65578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Pawe=C5=82czyk?= Date: Wed, 8 May 2024 17:33:43 +0200 Subject: [PATCH 020/137] ggml : expose SSE3 and SSSE3 for MSVC when AVX is available (whisper/2128) --- ggml-impl.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ggml-impl.h b/ggml-impl.h index d85b152bf..59684fa81 100644 --- a/ggml-impl.h +++ b/ggml-impl.h @@ -120,9 +120,16 @@ extern "C" { #ifndef __F16C__ #define __F16C__ #endif +#endif + +// __SSE3__ and __SSSE3__ are not defined in MSVC, but SSE3/SSSE3 are present when AVX/AVX2/AVX512 are available +#if defined(_MSC_VER) && (defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)) #ifndef __SSE3__ #define __SSE3__ #endif +#ifndef __SSSE3__ +#define __SSSE3__ +#endif #endif // 16-bit float From c3c88f296a72432edb697ac8026dbf2ec18f2b21 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 12 May 2024 20:36:31 +0300 Subject: [PATCH 021/137] ggml : try fix ppc64 (whisper/0) --- ggml-quants.c | 2 +- ggml.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ggml-quants.c b/ggml-quants.c index f711bd013..9e62a3f32 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -11425,7 +11425,7 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void vector signed short qxh = (vector signed short)vec_sld(vec_splats(qh[1]), vec_splats(qh[0]), 8); qh += 2; - vector bool short vsel = vec_cmpge(qxh, (vector signed short)v0); + vector __bool short vsel = vec_cmpge(qxh, (vector signed short)v0); vector signed short q8ysum = vec_sel((vector signed short)vec_xor((vector unsigned short)q8ysums, vsign), q8ysums, vsel); diff --git a/ggml.c b/ggml.c index b96a82a41..d443a9b42 100644 --- a/ggml.c +++ b/ggml.c @@ -1306,6 +1306,8 @@ static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) { #define GGML_F16_VEC_ZERO GGML_F32x4_ZERO #define GGML_F16_VEC_SET1 GGML_F32x4_SET1 #define GGML_F16_VEC_FMA GGML_F32x4_FMA +#define GGML_F16_VEC_ADD GGML_F32x4_ADD +#define GGML_F16_VEC_MUL GGML_F32x4_MUL #define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE // Use vec_xl, not vec_ld, in case the load address is not aligned. #define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \ From f308ea705974dff62a1fe5367d776ad9d5109239 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 13 May 2024 11:01:07 +0300 Subject: [PATCH 022/137] metal : tune soft_max number of threads (whisper/0) --- ggml-metal.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index 28dec762a..bfa352c3a 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -1378,7 +1378,7 @@ static enum ggml_status ggml_metal_graph_compute( const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16); if (ne00%4 == 0) { - while (nth < ne00/4 && nth < 256) { + while (nth < ne00/4 && nth*ne01*ne02*ne03 < 256) { nth *= 2; } if (use_f16) { @@ -1387,7 +1387,7 @@ static enum ggml_status ggml_metal_graph_compute( pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_SOFT_MAX_F32_4].pipeline; } } else { - while (nth < ne00 && nth < 1024) { + while (nth < ne00 && nth*ne01*ne02*ne03 < 256) { nth *= 2; } if (use_f16) { From a5e3fde8578d54b98d941344a4da150669af200d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 14 May 2024 15:33:16 +0300 Subject: [PATCH 023/137] sync : ggml ggml-ci --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 1ea320429..b2b5f8fc1 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -30f54cbb3ada3e4c5bc6924de3e5918e5be4ff11 +fafd5e7f89382b8cfb51e3dac8d4f1500ca44918 From e8a7fd4fb06d82f663850c21fcf86c0fb98ad9b4 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 14 May 2024 19:09:30 +0300 Subject: [PATCH 024/137] metal : support FA without mask + add asserts (#7278) * ggml : fa without mask + add asserts ggml-ci * metal : support non-contiguous KV ggml-ci --- ggml-metal.m | 69 +++++++++++++++++++++----------------- ggml-metal.metal | 53 +++++++++++------------------ ggml.c | 10 ++++++ ggml.h | 3 +- tests/test-backend-ops.cpp | 25 ++++++++------ 5 files changed, 85 insertions(+), 75 deletions(-) diff --git a/ggml-metal.m b/ggml-metal.m index bfa352c3a..390a1cd78 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -2512,13 +2512,14 @@ static enum ggml_status ggml_metal_graph_compute( } break; case GGML_OP_FLASH_ATTN_EXT: { - GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ne00 % 4 == 0); + GGML_ASSERT(ne11 % 32 == 0); + GGML_ASSERT(src0->type == GGML_TYPE_F32); - struct ggml_tensor * src3 = gf->nodes[i]->src[3]; + GGML_ASSERT(ggml_are_same_shape (src1, src2)); - GGML_ASSERT(ggml_are_same_shape(src1, src2)); - GGML_ASSERT(src3); + struct ggml_tensor * src3 = gf->nodes[i]->src[3]; size_t offs_src3 = 0; @@ -2528,6 +2529,11 @@ static enum ggml_status ggml_metal_graph_compute( GGML_ASSERT(!src3 || src3->ne[1] >= GGML_PAD(src0->ne[1], 8) && "the Flash-Attention Metal kernel requires the mask to be padded to 8 and at least n_queries big"); + const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20); + const uint64_t nb21 = src2 ? src2->nb[1] : 0; + const uint64_t nb22 = src2 ? src2->nb[2] : 0; + const uint64_t nb23 = src2 ? src2->nb[3] : 0; + const int64_t ne30 = src3 ? src3->ne[0] : 0; GGML_UNUSED(ne30); //const int64_t ne31 = src3 ? src3->ne[1] : 0; const int64_t ne32 = src3 ? src3->ne[2] : 0; GGML_UNUSED(ne32); @@ -2590,34 +2596,35 @@ static enum ggml_status ggml_metal_graph_compute( [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2]; - [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3]; + if (id_src3) { + [encoder setBuffer:id_src3 offset:offs_src3 atIndex:3]; + } else { + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:3]; + } [encoder setBuffer:id_dst offset:offs_dst atIndex:4]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:5]; - [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:6]; - [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:7]; - [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:8]; - [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:9]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:10]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:11]; - [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:12]; - [encoder setBytes:&ne10 length:sizeof( int64_t) atIndex:13]; - [encoder setBytes:&ne11 length:sizeof( int64_t) atIndex:14]; - [encoder setBytes:&ne12 length:sizeof( int64_t) atIndex:15]; - [encoder setBytes:&ne13 length:sizeof( int64_t) atIndex:16]; - [encoder setBytes:&nb10 length:sizeof(uint64_t) atIndex:17]; - [encoder setBytes:&nb11 length:sizeof(uint64_t) atIndex:18]; - [encoder setBytes:&nb12 length:sizeof(uint64_t) atIndex:19]; - [encoder setBytes:&nb13 length:sizeof(uint64_t) atIndex:20]; - [encoder setBytes:&nb31 length:sizeof(uint64_t) atIndex:21]; - [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:22]; - [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:23]; - [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:24]; - [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:25]; - [encoder setBytes:&scale length:sizeof( float) atIndex:26]; - [encoder setBytes:&max_bias length:sizeof( float) atIndex:27]; - [encoder setBytes:&m0 length:sizeof(m0) atIndex:28]; - [encoder setBytes:&m1 length:sizeof(m1) atIndex:29]; - [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:30]; + [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:5]; + [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:6]; + [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:7]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8]; + [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9]; + [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10]; + [encoder setBytes:&ne11 length:sizeof( int64_t) atIndex:11]; + [encoder setBytes:&ne12 length:sizeof( int64_t) atIndex:12]; + [encoder setBytes:&ne13 length:sizeof( int64_t) atIndex:13]; + [encoder setBytes:&nb11 length:sizeof(uint64_t) atIndex:14]; + [encoder setBytes:&nb12 length:sizeof(uint64_t) atIndex:15]; + [encoder setBytes:&nb13 length:sizeof(uint64_t) atIndex:16]; + [encoder setBytes:&nb21 length:sizeof(uint64_t) atIndex:17]; + [encoder setBytes:&nb22 length:sizeof(uint64_t) atIndex:18]; + [encoder setBytes:&nb23 length:sizeof(uint64_t) atIndex:19]; + [encoder setBytes:&nb31 length:sizeof(uint64_t) atIndex:20]; + [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:21]; + [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:22]; + [encoder setBytes:&scale length:sizeof( float) atIndex:23]; + [encoder setBytes:&max_bias length:sizeof( float) atIndex:24]; + [encoder setBytes:&m0 length:sizeof(m0) atIndex:25]; + [encoder setBytes:&m1 length:sizeof(m1) atIndex:26]; + [encoder setBytes:&n_head_log2 length:sizeof(n_head_log2) atIndex:27]; if (!use_vec_kernel) { // half8x8 kernel diff --git a/ggml-metal.metal b/ggml-metal.metal index 7af4e8f93..57fdf564e 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2049,27 +2049,24 @@ typedef void (flash_attn_ext_f16_t)( device const char * v, device const char * mask, device float * dst, - constant int64_t & ne00, constant int64_t & ne01, constant int64_t & ne02, constant int64_t & ne03, - constant uint64_t & nb00, constant uint64_t & nb01, constant uint64_t & nb02, constant uint64_t & nb03, - constant int64_t & ne10, constant int64_t & ne11, constant int64_t & ne12, constant int64_t & ne13, - constant uint64_t & nb10, constant uint64_t & nb11, constant uint64_t & nb12, constant uint64_t & nb13, + constant uint64_t & nb21, + constant uint64_t & nb22, + constant uint64_t & nb23, constant uint64_t & nb31, - constant int64_t & ne0, constant int64_t & ne1, constant int64_t & ne2, - constant int64_t & ne3, constant float & scale, constant float & max_bias, constant float & m0, @@ -2090,27 +2087,24 @@ kernel void kernel_flash_attn_ext_f16( device const char * v, device const char * mask, device float * dst, - constant int64_t & ne00, constant int64_t & ne01, constant int64_t & ne02, constant int64_t & ne03, - constant uint64_t & nb00, constant uint64_t & nb01, constant uint64_t & nb02, constant uint64_t & nb03, - constant int64_t & ne10, constant int64_t & ne11, constant int64_t & ne12, constant int64_t & ne13, - constant uint64_t & nb10, constant uint64_t & nb11, constant uint64_t & nb12, constant uint64_t & nb13, + constant uint64_t & nb21, + constant uint64_t & nb22, + constant uint64_t & nb23, constant uint64_t & nb31, - constant int64_t & ne0, constant int64_t & ne1, constant int64_t & ne2, - constant int64_t & ne3, constant float & scale, constant float & max_bias, constant float & m0, @@ -2180,10 +2174,6 @@ kernel void kernel_flash_attn_ext_f16( const short ne22 = ne12; const short ne23 = ne13; - const uint nb21 = nb11; - const uint nb22 = nb12; - const uint nb23 = nb13; - // broadcast const short rk2 = ne02/ne12; const short rk3 = ne03/ne13; @@ -2247,11 +2237,16 @@ kernel void kernel_flash_attn_ext_f16( simdgroup_multiply_accumulate(mqk, mq[i], mk, mqk); } - // mqk = mqk*scale + mask*slope - simdgroup_half8x8 mm; - simdgroup_load(mm, mp + ic + 8*cc, nb31/sizeof(half), 0, false); - simdgroup_multiply(mm, mslope, mm); - simdgroup_multiply_accumulate(mqk, mqk, mscale, mm); + if (mask != q) { + // mqk = mqk*scale + mask*slope + simdgroup_half8x8 mm; + simdgroup_load(mm, mp + ic + 8*cc, nb31/sizeof(half), 0, false); + simdgroup_multiply(mm, mslope, mm); + simdgroup_multiply_accumulate(mqk, mqk, mscale, mm); + } else { + // mqk = mqk*scale + simdgroup_multiply(mqk, mscale, mqk); + } simdgroup_store(mqk, ss + 8*cc, TF, 0, false); } @@ -2425,27 +2420,24 @@ kernel void kernel_flash_attn_ext_vec_f16( device const char * v, device const char * mask, device float * dst, - constant int64_t & ne00, constant int64_t & ne01, constant int64_t & ne02, constant int64_t & ne03, - constant uint64_t & nb00, constant uint64_t & nb01, constant uint64_t & nb02, constant uint64_t & nb03, - constant int64_t & ne10, constant int64_t & ne11, constant int64_t & ne12, constant int64_t & ne13, - constant uint64_t & nb10, constant uint64_t & nb11, constant uint64_t & nb12, constant uint64_t & nb13, + constant uint64_t & nb21, + constant uint64_t & nb22, + constant uint64_t & nb23, constant uint64_t & nb31, - constant int64_t & ne0, constant int64_t & ne1, constant int64_t & ne2, - constant int64_t & ne3, constant float & scale, constant float & max_bias, constant float & m0, @@ -2521,10 +2513,6 @@ kernel void kernel_flash_attn_ext_vec_f16( const short ne22 = ne12; const short ne23 = ne13; - const uint nb21 = nb11; - const uint nb22 = nb12; - const uint nb23 = nb13; - // broadcast const short rk2 = ne02/ne12; const short rk3 = ne03/ne13; @@ -2589,8 +2577,7 @@ kernel void kernel_flash_attn_ext_vec_f16( // mqk = mqk*scale + mask*slope if (tiisg == 0) { - float4 mm = (float4) mp4[ic/4 + cc]; - mqk = mqk*scale + mm*slope; + mqk = mqk*scale + ((mask != q) ? ((float4) mp4[ic/4 + cc])*slope : (float4) 0.0f); ss4[cc] = mqk; } diff --git a/ggml.c b/ggml.c index d443a9b42..03b609ddd 100644 --- a/ggml.c +++ b/ggml.c @@ -2824,6 +2824,16 @@ bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor (t0->ne[3] == t1->ne[3] ); } +bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { + static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); + + return + (t0->nb[0] == t1->nb[0] ) && + (t0->nb[1] == t1->nb[1] ) && + (t0->nb[2] == t1->nb[2] ) && + (t0->nb[3] == t1->nb[3] ); +} + // check if t1 can be represented as a repeatition of t0 static inline bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) { static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function"); diff --git a/ggml.h b/ggml.h index 3fe95ed57..25f4f73a8 100644 --- a/ggml.h +++ b/ggml.h @@ -766,7 +766,8 @@ extern "C" { GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor); GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars - GGML_API bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1); + GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1); // use this to compute the memory overhead of a tensor GGML_API size_t ggml_tensor_overhead(void); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 45a2cb85a..f080f7e22 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1487,25 +1487,27 @@ struct test_flash_attn_ext : public test_case { const int64_t kv; // kv size const int64_t nb; // batch size + const bool mask; // use mask + const float max_bias; // ALiBi std::string vars() override { - return VARS_TO_STR5(hs, nh, kv, nb, max_bias); + return VARS_TO_STR6(hs, nh, kv, nb, mask, max_bias); } double max_nmse_err() override { return 5e-4; } - test_flash_attn_ext(int64_t hs = 128, int64_t nh = 32, int64_t kv = 96, int64_t nb = 8, float max_bias = 0.0f) - : hs(hs), nh(nh), kv(kv), nb(nb), max_bias(max_bias) {} + test_flash_attn_ext(int64_t hs = 128, int64_t nh = 32, int64_t kv = 96, int64_t nb = 8, bool mask = true, float max_bias = 0.0f) + : hs(hs), nh(nh), kv(kv), nb(nb), mask(mask), max_bias(max_bias) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, hs, nb, nh, 1); ggml_tensor * k = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, hs, kv, nh, 1); ggml_tensor * v = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, hs, kv, nh, 1); - ggml_tensor * mask = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, GGML_PAD(nb, GGML_KQ_MASK_PAD), 1, 1); - ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, mask, 1.0f/sqrtf(hs), max_bias); + ggml_tensor * m = mask ? ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, GGML_PAD(nb, GGML_KQ_MASK_PAD), 1, 1) : nullptr; + ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hs), max_bias); return out; } }; @@ -2175,11 +2177,14 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_leaky_relu()); for (int hs : { 64, 80, 128, 256, }) { - for (float max_bias : {0.0f, 8.0f}) { - for (int nh : { 32, }) { - for (int kv : { 512, 1024, }) { - for (int nb : { 1, 2, 4, 8, }) { - test_cases.emplace_back(new test_flash_attn_ext(hs, nh, kv, nb, max_bias)); + for (bool mask : { true, false } ) { + for (float max_bias : { 0.0f, 8.0f }) { + if (!mask && max_bias > 0.0f) continue; + for (int nh : { 32, }) { + for (int kv : { 512, 1024, }) { + for (int nb : { 1, 2, 4, 8, }) { + test_cases.emplace_back(new test_flash_attn_ext(hs, nh, kv, nb, mask, max_bias)); + } } } } From 9f773486ab78d65f5cca3f7e31c862b7043bf721 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 14 May 2024 19:14:38 +0300 Subject: [PATCH 025/137] script : sync ggml-rpc --- scripts/sync-ggml-am.sh | 4 ++++ scripts/sync-ggml.sh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/scripts/sync-ggml-am.sh b/scripts/sync-ggml-am.sh index 7c157eb4c..cf22afc41 100755 --- a/scripts/sync-ggml-am.sh +++ b/scripts/sync-ggml-am.sh @@ -112,6 +112,8 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then # src/ggml-opencl.h -> ggml-opencl.h # src/ggml-quants.c -> ggml-quants.c # src/ggml-quants.h -> ggml-quants.h + # src/ggml-rpc.cpp -> ggml-rpc.cpp + # src/ggml-rpc.h -> ggml-rpc.h # src/ggml-sycl.cpp -> ggml-sycl.cpp # src/ggml-sycl.h -> ggml-sycl.h # src/ggml-vulkan.cpp -> ggml-vulkan.cpp @@ -149,6 +151,8 @@ if [ -f $SRC_LLAMA/ggml-src.patch ]; then -e 's/src\/ggml-opencl\.h/ggml-opencl.h/g' \ -e 's/src\/ggml-quants\.c/ggml-quants.c/g' \ -e 's/src\/ggml-quants\.h/ggml-quants.h/g' \ + -e 's/src\/ggml-rpc\.cpp/ggml-rpc.cpp/g' \ + -e 's/src\/ggml-rpc\.h/ggml-rpc.h/g' \ -e 's/src\/ggml-sycl\.cpp/ggml-sycl.cpp/g' \ -e 's/src\/ggml-sycl\.h/ggml-sycl.h/g' \ -e 's/src\/ggml-vulkan\.cpp/ggml-vulkan.cpp/g' \ diff --git a/scripts/sync-ggml.sh b/scripts/sync-ggml.sh index 80c43976f..ec47fb27c 100755 --- a/scripts/sync-ggml.sh +++ b/scripts/sync-ggml.sh @@ -20,6 +20,8 @@ cp -rpv ../ggml/src/ggml-opencl.cpp ./ggml-opencl.cpp cp -rpv ../ggml/src/ggml-opencl.h ./ggml-opencl.h cp -rpv ../ggml/src/ggml-quants.c ./ggml-quants.c cp -rpv ../ggml/src/ggml-quants.h ./ggml-quants.h +cp -rpv ../ggml/src/ggml-rpc.cpp ./ggml-rpc.cpp +cp -rpv ../ggml/src/ggml-rpc.h ./ggml-rpc.h cp -rpv ../ggml/src/ggml-sycl.cpp ./ggml-sycl.cpp cp -rpv ../ggml/src/ggml-sycl.h ./ggml-sycl.h cp -rpv ../ggml/src/ggml-vulkan.cpp ./ggml-vulkan.cpp From 583fd6b000ec9ad1b465b5c98524f4a0ae388077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Wed, 15 May 2024 08:44:16 +0200 Subject: [PATCH 026/137] server bench: fix bench not waiting for model load (#7284) --- examples/server/bench/bench.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/server/bench/bench.py b/examples/server/bench/bench.py index 86c5de101..25ac29c4c 100644 --- a/examples/server/bench/bench.py +++ b/examples/server/bench/bench.py @@ -293,13 +293,14 @@ def start_server_background(args): def is_server_listening(server_fqdn, server_port): - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - result = sock.connect_ex((server_fqdn, server_port)) - _is_server_listening = result == 0 - if _is_server_listening: - print(f"server is listening on {server_fqdn}:{server_port}...") - return _is_server_listening - + try: + url = f"{server_fqdn}:{server_port}/health" + if not url.startswith("http://"): + url = f"http://{url}" + result = requests.get(url) + return result.status_code == 200 + except Exception: + return False def escape_metric_name(metric_name): return re.sub('[^A-Z0-9]', '_', metric_name.upper()) From 48aa8fd1f213a69b41569f809cc954f24dbc4366 Mon Sep 17 00:00:00 2001 From: John Balis Date: Wed, 15 May 2024 03:52:33 -0500 Subject: [PATCH 027/137] ggml : add `ggml_upscale_ext` (ggml/814) * initial commit with CPU implementation of upscale to shape and test, cuda implementation next * experimental commit to see if dst shape is correct * test version * test * removed unnecessary params * refactor * fixed tests * ggml : metal impl + cleanup + sycl dev warnings * patched ggml_upscale cuda op to handle non-contiguous tensors, added test for non-contiguous behavior * metal : fix upsacle op to support nb00 + style --------- Co-authored-by: Georgi Gerganov --- ggml-cuda/upscale.cu | 63 +++++++++++++++++++------------------ ggml-metal.m | 10 ++++-- ggml-metal.metal | 21 ++++++++----- ggml-sycl.cpp | 4 +++ ggml.c | 64 ++++++++++++++++++++++++++++---------- ggml.h | 12 +++++++ tests/test-backend-ops.cpp | 32 +++++++++++++++++-- 7 files changed, 146 insertions(+), 60 deletions(-) diff --git a/ggml-cuda/upscale.cu b/ggml-cuda/upscale.cu index 2f62fed48..cf513c3ad 100644 --- a/ggml-cuda/upscale.cu +++ b/ggml-cuda/upscale.cu @@ -1,35 +1,36 @@ #include "upscale.cuh" -static __global__ void upscale_f32(const float * x, float * dst, const int ne00, const int ne00xne01, const int scale_factor) { - // blockIdx.z: idx of ne02*ne03 - // blockIdx.y: idx of ne01*scale_factor， aka ne1 - // blockIDx.x: idx of ne00*scale_factor / BLOCK_SIZE - // ne00xne01: ne00 * ne01 - int ne0 = ne00 * scale_factor; - int nidx = threadIdx.x + blockIdx.x * blockDim.x; - if (nidx >= ne0) { +static __global__ void upscale_f32(const float * x, float * dst, + const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int ne13, + const float sf0, const float sf1, const float sf2, const float sf3) { + int index = threadIdx.x + blockIdx.x * blockDim.x; + if (index >= ne10 * ne11 * ne12 * ne13) { return; } - // operation - int i00 = nidx / scale_factor; - int i01 = blockIdx.y / scale_factor; - int offset_src = - i00 + - i01 * ne00 + - blockIdx.z * ne00xne01; - int offset_dst = - nidx + - blockIdx.y * ne0 + - blockIdx.z * ne0 * gridDim.y; - dst[offset_dst] = x[offset_src]; + + int i10 = index % ne10; + int i11 = (index / ne10) % ne11; + int i12 = (index / (ne10 * ne11)) % ne12; + int i13 = (index / (ne10 * ne11 * ne12)) % ne13; + + int i00 = i10 / sf0; + int i01 = i11 / sf1; + int i02 = i12 / sf2; + int i03 = i13 / sf3; + + dst[index] = *(float *)((char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00); } -static void upscale_f32_cuda(const float * x, float * dst, const int ne00, const int ne01, const int ne02, const int ne03, - const int scale_factor, cudaStream_t stream) { - int ne0 = (ne00 * scale_factor); - int num_blocks = (ne0 + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE; - dim3 gridDim(num_blocks, (ne01 * scale_factor), ne02*ne03); - upscale_f32<<>>(x, dst, ne00, ne00 * ne01, scale_factor); +static void upscale_f32_cuda(const float * x, float * dst, + const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int ne13, + const float sf0, const float sf1, const float sf2, const float sf3, + cudaStream_t stream) { + int dst_size = ne10 * ne11 * ne12 * ne13; + int num_blocks = (dst_size + CUDA_UPSCALE_BLOCK_SIZE - 1) / CUDA_UPSCALE_BLOCK_SIZE; + + upscale_f32<<>>(x, dst, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3); } void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { @@ -39,10 +40,12 @@ void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { cudaStream_t stream = ctx.stream(); GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); - GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors + GGML_ASSERT( dst->type == GGML_TYPE_F32); - const int scale_factor = dst->op_params[0]; + const float sf0 = (float)dst->ne[0]/src0->ne[0]; + const float sf1 = (float)dst->ne[1]/src0->ne[1]; + const float sf2 = (float)dst->ne[2]/src0->ne[2]; + const float sf3 = (float)dst->ne[3]/src0->ne[3]; - upscale_f32_cuda(src0_d, dst_d, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], scale_factor, stream); + upscale_f32_cuda(src0_d, dst_d, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3, stream); } diff --git a/ggml-metal.m b/ggml-metal.m index 390a1cd78..b0b16dbf7 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -2353,7 +2353,10 @@ static enum ggml_status ggml_metal_graph_compute( { GGML_ASSERT(src0->type == GGML_TYPE_F32); - const int sf = dst->op_params[0]; + const float sf0 = (float)ne0/src0->ne[0]; + const float sf1 = (float)ne1/src0->ne[1]; + const float sf2 = (float)ne2/src0->ne[2]; + const float sf3 = (float)ne3/src0->ne[3]; const id pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_UPSCALE_F32].pipeline; @@ -2376,7 +2379,10 @@ static enum ggml_status ggml_metal_graph_compute( [encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15]; [encoder setBytes:&nb2 length:sizeof(nb2) atIndex:16]; [encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17]; - [encoder setBytes:&sf length:sizeof(sf) atIndex:18]; + [encoder setBytes:&sf0 length:sizeof(sf0) atIndex:18]; + [encoder setBytes:&sf1 length:sizeof(sf1) atIndex:19]; + [encoder setBytes:&sf2 length:sizeof(sf2) atIndex:20]; + [encoder setBytes:&sf3 length:sizeof(sf3) atIndex:21]; const int nth = MIN((int) pipeline.maxTotalThreadsPerThreadgroup, ne0); diff --git a/ggml-metal.metal b/ggml-metal.metal index 57fdf564e..386e9195f 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -1852,7 +1852,10 @@ kernel void kernel_upscale_f32( constant uint64_t & nb1, constant uint64_t & nb2, constant uint64_t & nb3, - constant int32_t & sf, + constant float & sf0, + constant float & sf1, + constant float & sf2, + constant float & sf3, uint3 tgpig[[threadgroup_position_in_grid]], uint3 tpitg[[thread_position_in_threadgroup]], uint3 ntg[[threads_per_threadgroup]]) { @@ -1861,15 +1864,17 @@ kernel void kernel_upscale_f32( const int64_t i2 = tgpig.y; const int64_t i1 = tgpig.x; - const int64_t i03 = i3; - const int64_t i02 = i2; - const int64_t i01 = i1/sf; - - device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01); - device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1); + const int64_t i03 = i3/sf3; + const int64_t i02 = i2/sf2; + const int64_t i01 = i1/sf1; for (int i0 = tpitg.x; i0 < ne0; i0 += ntg.x) { - dst_ptr[i0] = src0_ptr[i0/sf]; + const int64_t i00 = i0/sf0; + + device const float * src0_ptr = (device const float *) (src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00); + device float * dst_ptr = (device float *) (dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + + dst_ptr[0] = src0_ptr[0]; } } diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 724070eb9..b15efb704 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -13987,6 +13987,10 @@ inline void ggml_sycl_op_upscale(const ggml_tensor *src0, GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors +#pragma message("TODO: generalize upscale operator") +#pragma message(" https://github.com/ggerganov/ggml/pull/814") + GGML_ASSERT(false && "TODO: generalize upscale operator); + const int scale_factor = dst->op_params[0]; upscale_f32_sycl(src0_dd, dst_dd, src0->ne[0], src0->ne[1], src0->ne[2], scale_factor, main_stream); diff --git a/ggml.c b/ggml.c index 03b609ddd..f09cc3060 100644 --- a/ggml.c +++ b/ggml.c @@ -6293,7 +6293,10 @@ struct ggml_tensor * ggml_pool_2d( static struct ggml_tensor * ggml_upscale_impl( struct ggml_context * ctx, struct ggml_tensor * a, - int scale_factor) { + int ne0, + int ne1, + int ne2, + int ne3) { bool is_node = false; if (a->grad) { @@ -6301,19 +6304,45 @@ static struct ggml_tensor * ggml_upscale_impl( is_node = true; } + GGML_ASSERT(a->ne[0] <= ne0); + GGML_ASSERT(a->ne[1] <= ne1); + GGML_ASSERT(a->ne[2] <= ne2); + GGML_ASSERT(a->ne[3] <= ne3); + struct ggml_tensor * result = ggml_new_tensor_4d(ctx, a->type, - a->ne[0] * scale_factor, - a->ne[1] * scale_factor, - a->ne[2], a->ne[3]); + ne0, + ne1, + ne2, + ne3 + ); result->op = GGML_OP_UPSCALE; - result->op_params[0] = scale_factor; + result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; result->src[0] = a; return result; } +struct ggml_tensor * ggml_upscale( + struct ggml_context * ctx, + struct ggml_tensor * a, + int scale_factor) { + return ggml_upscale_impl(ctx, a, a->ne[0] * scale_factor, a->ne[1] * scale_factor, a->ne[2], a->ne[3]); +} + +struct ggml_tensor * ggml_upscale_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int ne0, + int ne1, + int ne2, + int ne3) { + return ggml_upscale_impl(ctx, a, ne0, ne1, ne2, ne3); +} + +// ggml_pad + struct ggml_tensor * ggml_pad( struct ggml_context * ctx, struct ggml_tensor * a, @@ -6338,12 +6367,7 @@ struct ggml_tensor * ggml_pad( return result; } -struct ggml_tensor * ggml_upscale( - struct ggml_context * ctx, - struct ggml_tensor * a, - int scale_factor) { - return ggml_upscale_impl(ctx, a, scale_factor); -} +// ggml_arange struct ggml_tensor * ggml_arange( struct ggml_context * ctx, @@ -6365,6 +6389,8 @@ struct ggml_tensor * ggml_arange( return result; } +// ggml_timestep_embedding + struct ggml_tensor * ggml_timestep_embedding( struct ggml_context * ctx, struct ggml_tensor * timesteps, @@ -14820,25 +14846,28 @@ static void ggml_compute_forward_upscale_f32( return; } - GGML_ASSERT(src0->nb[0] == sizeof(float)); + GGML_ASSERT(src0->type == GGML_TYPE_F32); const int ith = params->ith; const int nth = params->nth; GGML_TENSOR_UNARY_OP_LOCALS - const int scale_factor = dst->op_params[0]; + const float sf0 = (float)ne0/src0->ne[0]; + const float sf1 = (float)ne1/src0->ne[1]; + const float sf2 = (float)ne2/src0->ne[2]; + const float sf3 = (float)ne3/src0->ne[3]; // TODO: optimize for (int64_t i3 = 0; i3 < ne3; i3++) { - const int64_t i03 = i3; + const int64_t i03 = i3 / sf3; for (int64_t i2 = ith; i2 < ne2; i2 += nth) { - const int64_t i02 = i2; + const int64_t i02 = i2 / sf2; for (int64_t i1 = 0; i1 < ne1; i1++) { - const int64_t i01 = i1 / scale_factor; + const int64_t i01 = i1 / sf1; for (int64_t i0 = 0; i0 < ne0; i0++) { - const int64_t i00 = i0 / scale_factor; + const int64_t i00 = i0 / sf0; const float * x = (float *)((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); float * y = (float *)((char *) dst->data + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); @@ -14868,6 +14897,7 @@ static void ggml_compute_forward_upscale( } } + // ggml_compute_forward_pad static void ggml_compute_forward_pad_f32( diff --git a/ggml.h b/ggml.h index 25f4f73a8..5e121604a 100644 --- a/ggml.h +++ b/ggml.h @@ -1674,12 +1674,24 @@ extern "C" { float p1); // nearest interpolate + // multiplies ne0 and ne1 by scale factor // used in stable-diffusion GGML_API struct ggml_tensor * ggml_upscale( struct ggml_context * ctx, struct ggml_tensor * a, int scale_factor); + // nearest interpolate + // nearest interpolate to specified dimensions + // used in tortoise.cpp + GGML_API struct ggml_tensor * ggml_upscale_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int ne0, + int ne1, + int ne2, + int ne3); + // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0] GGML_API struct ggml_tensor * ggml_pad( struct ggml_context * ctx, diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f080f7e22..85ef21c2a 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1329,23 +1329,47 @@ struct test_upscale : public test_case { const ggml_type type; const std::array ne; const int32_t scale_factor; + const bool transpose; std::string vars() override { - return VARS_TO_STR3(type, ne, scale_factor); + return VARS_TO_STR4(type, ne, scale_factor, transpose); } test_upscale(ggml_type type = GGML_TYPE_F32, std::array ne = {512, 512, 3, 1}, - int32_t scale_factor = 2) - : type(type), ne(ne), scale_factor(scale_factor) {} + int32_t scale_factor = 2, bool transpose = false) + : type(type), ne(ne), scale_factor(scale_factor), transpose(transpose) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + if (transpose) a = ggml_transpose(ctx, a); ggml_tensor * out = ggml_upscale(ctx, a, scale_factor); return out; } }; +// GGML_OP_UPSCALE (ext) +struct test_upscale_ext : public test_case { + const ggml_type type; + const std::array ne; + const std::array ne_tgt; + + std::string vars() override { + return VARS_TO_STR3(type, ne, ne_tgt); + } + + test_upscale_ext(ggml_type type = GGML_TYPE_F32, + std::array ne = {2, 5, 7, 11}, + std::array ne_tgt = {5, 7, 11, 13}) + : type(type), ne(ne), ne_tgt(ne_tgt) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * out = ggml_upscale_ext(ctx, a, ne_tgt[0], ne_tgt[1],ne_tgt[2], ne_tgt[3]); + return out; + } +}; + // GGML_OP_GROUP_NORM struct test_group_norm : public test_case { const ggml_type type; @@ -2169,6 +2193,8 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_sum_rows()); test_cases.emplace_back(new test_upscale()); + test_cases.emplace_back(new test_upscale(GGML_TYPE_F32, { 512, 512, 3, 1 }, 2, true)); + test_cases.emplace_back(new test_upscale_ext()); test_cases.emplace_back(new test_group_norm()); test_cases.emplace_back(new test_acc()); test_cases.emplace_back(new test_pad()); From 29499bb59383c2a8c5d557a90abb08b696cef7f6 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 15 May 2024 13:23:41 +0300 Subject: [PATCH 028/137] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index b2b5f8fc1..57bede67b 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -fafd5e7f89382b8cfb51e3dac8d4f1500ca44918 +126d34985705a5a2222723c145cb4e125ac689f3 From ea3b0590ee33d3573eb8ef76f88cc60f36d2a38d Mon Sep 17 00:00:00 2001 From: dm4 Date: Wed, 15 May 2024 20:01:12 +0800 Subject: [PATCH 029/137] embedding : free the batch after execution (#7297) --- examples/embedding/embedding.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index c85a2da53..0c921ed69 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -211,6 +211,7 @@ int main(int argc, char ** argv) { // clean up llama_print_timings(ctx); + llama_batch_free(batch); llama_free(ctx); llama_free_model(model); llama_backend_free(); From 9a17ab914b0aa7353389c656a3f2a0f086726868 Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Wed, 15 May 2024 13:26:30 +0100 Subject: [PATCH 030/137] Add missing " (#7303) --- ggml-sycl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index b15efb704..19d22d637 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -13989,7 +13989,7 @@ inline void ggml_sycl_op_upscale(const ggml_tensor *src0, #pragma message("TODO: generalize upscale operator") #pragma message(" https://github.com/ggerganov/ggml/pull/814") - GGML_ASSERT(false && "TODO: generalize upscale operator); + GGML_ASSERT(false && "TODO: generalize upscale operator"); const int scale_factor = dst->op_params[0]; From 344f9126cc0d15891fde9472fe40b8572628ad7d Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 15 May 2024 15:08:48 +0200 Subject: [PATCH 031/137] ggml : tag ggml_tensor::backend as deprecated (#7290) --- examples/llava/llava.cpp | 15 --------------- ggml-backend.c | 1 - ggml.c | 10 ++++++++++ ggml.h | 3 ++- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/examples/llava/llava.cpp b/examples/llava/llava.cpp index 9a990bb18..63878d176 100644 --- a/examples/llava/llava.cpp +++ b/examples/llava/llava.cpp @@ -88,7 +88,6 @@ static struct clip_image_grid_shape get_anyres_image_grid_shape(const std::pair< // Take the image segments in a grid configuration and return the embeddings and the number of embeddings into preallocated memory (image_embd_out) static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector & image_embd_v, struct clip_image_grid_shape grid_shape, float * image_embd_out, int * n_img_pos_out) { struct { - struct ggml_tensor * newline; struct ggml_context * ctx; } model; @@ -150,20 +149,6 @@ static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector model.ctx = ggml_init(params); - ggml_tensor * newline_tmp = clip_get_newline_tensor(ctx_clip); - model.newline = ggml_new_tensor_1d(model.ctx, GGML_TYPE_F32, newline_tmp->ne[0]); - if (newline_tmp->backend != GGML_BACKEND_TYPE_CPU) { - if (newline_tmp->buffer == NULL) { - LOG_TEE("newline_tmp tensor buffer is NULL\n"); - } - ggml_backend_tensor_get(newline_tmp, model.newline->data, 0, ggml_nbytes(newline_tmp)); - } else { - model.newline->data = newline_tmp->data; - if (model.newline->data == NULL) { - LOG_TEE("newline_tmp tensor data is NULL\n"); - } - } - struct ggml_tensor * image_features = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, clip_n_mmproj_embd(ctx_clip), clip_n_patches(ctx_clip), num_images - 1); // example: 4096 x 576 x 4 // ggml_tensor_printf(image_features,"image_features",__LINE__,false,false); // fill it with the image embeddings, ignoring the base diff --git a/ggml-backend.c b/ggml-backend.c index dd090a583..9e35ce98d 100644 --- a/ggml-backend.c +++ b/ggml-backend.c @@ -1895,7 +1895,6 @@ void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * t tensor->buffer = buffer; tensor->data = (char *)tensor->view_src->data + tensor->view_offs; - tensor->backend = tensor->view_src->backend; ggml_backend_buffer_init_tensor(buffer, tensor); } diff --git a/ggml.c b/ggml.c index f09cc3060..67e17a210 100644 --- a/ggml.c +++ b/ggml.c @@ -3178,6 +3178,12 @@ static struct ggml_tensor * ggml_new_tensor_impl( struct ggml_tensor * const result = (struct ggml_tensor *)((char *)ctx->mem_buffer + obj_new->offs); +#ifdef __clang__ + // temporary until ggml_tensor::backend is removed + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + *result = (struct ggml_tensor) { /*.type =*/ type, /*.backend =*/ GGML_BACKEND_TYPE_CPU, @@ -3200,6 +3206,10 @@ static struct ggml_tensor * ggml_new_tensor_impl( /*.padding =*/ { 0 }, }; +#ifdef __clang__ + #pragma clang diagnostic pop +#endif + // TODO: this should not be needed as long as we don't rely on aligned SIMD loads //ggml_assert_aligned(result->data); diff --git a/ggml.h b/ggml.h index 5e121604a..8c13f4ba8 100644 --- a/ggml.h +++ b/ggml.h @@ -565,7 +565,8 @@ extern "C" { // n-dimensional tensor struct ggml_tensor { enum ggml_type type; - enum ggml_backend_type backend; + + GGML_DEPRECATED(enum ggml_backend_type backend, "use the buffer type to find the storage location of the tensor"); struct ggml_backend_buffer * buffer; From dc020985b8755dd6aa93a2f002f43c3ede808cce Mon Sep 17 00:00:00 2001 From: agray3 Date: Wed, 15 May 2024 14:44:49 +0100 Subject: [PATCH 032/137] Avoid unnecessarily disabling CUDA graphs (#7302) As discussed in PR #6766, CUDA graphs were being disabled in the presence of long prompts. This fixes the issue by avoiding the consective update counter from incrementing unnecessarily for tokens in which cuda graphs are disabled due to batch size > 1. --- ggml-cuda.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 75a2ad480..04b6e5285 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -2558,7 +2558,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t } // Disable CUDA graphs (from the next token) if the use-case is demanding too many consecutive graph updates. - if (cuda_graph_update_required) { + if (use_cuda_graph && cuda_graph_update_required) { cuda_ctx->cuda_graph->number_consecutive_updates++; } else { cuda_ctx->cuda_graph->number_consecutive_updates = 0; From e1b40ac3b94824d761b5e26ea1bc5692706029d9 Mon Sep 17 00:00:00 2001 From: kunnis Date: Wed, 15 May 2024 12:59:12 -0500 Subject: [PATCH 033/137] ggml : use dynamic thread scheduling for matrix multiplication (#6915) * Just reordering some structs. * Adding in the calls to mm_pause * Passing around the state * Renaming and moving a bunch of variables around. * Extracting the logic to it's own function. * Moving some variable definitions into the chunk function. * Moving some variables around * moving src1_cont inside * Moving row_size * adding the current_chunk * Reorg the code. * Formatting to match the orig patch * starting to setup the chunking variables * Starting the buildup of the loop * The yield shouldn't be necessary. * adding the looping structure based on the chunk configuration. * Add in the re-chunking code. * Making it much more likely to rechunk. * disable resizing if numa is enabled. * Updating comments with what we've learned. * Fix formatting * Couple more formatting fixes. * More style fixes. * Fix Warnings * Going with unused because there's conditional logic that needs it. * Update ggml.c * Update ggml.c --------- --- ggml.c | 381 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 237 insertions(+), 144 deletions(-) diff --git a/ggml.c b/ggml.c index 67e17a210..684d84235 100644 --- a/ggml.c +++ b/ggml.c @@ -112,6 +112,8 @@ typedef void * thread_ret_t; #endif +typedef pthread_t ggml_thread_t; + #ifdef GGML_USE_CPU_HBM #include #endif @@ -1539,6 +1541,59 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) { #define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) #endif +// +// ggml context +// + +struct ggml_context { + size_t mem_size; + void* mem_buffer; + bool mem_buffer_owned; + bool no_alloc; + bool no_alloc_save; // this is used to save the no_alloc state when using scratch buffers + + int n_objects; + + struct ggml_object* objects_begin; + struct ggml_object* objects_end; + + struct ggml_scratch scratch; + struct ggml_scratch scratch_save; +}; + +struct ggml_context_container { + bool used; + + struct ggml_context context; +}; + +struct ggml_compute_state_shared { + const struct ggml_cgraph* cgraph; + const struct ggml_cplan* cplan; + + int64_t perf_node_start_cycles; + int64_t perf_node_start_time_us; + + const int n_threads; + + // synchronization primitives + atomic_int n_active; // num active threads + atomic_int node_n; // active graph node + atomic_int node_task; // active graph node task phase + + ggml_abort_callback abort_callback; // abort ggml_graph_compute when true + void* abort_callback_data; + + atomic_int current_chunk; // currently processing chunk during Mat_Mul, shared between all the threads. +}; + +struct ggml_compute_state { + ggml_thread_t thrd; + int ith; + struct ggml_compute_state_shared* shared; + enum ggml_status ec; +}; + // // fundamental operations // @@ -2385,32 +2440,6 @@ static void ggml_setup_op_has_task_pass(void) { } } -// -// ggml context -// - -struct ggml_context { - size_t mem_size; - void * mem_buffer; - bool mem_buffer_owned; - bool no_alloc; - bool no_alloc_save; // this is used to save the no_alloc state when using scratch buffers - - int n_objects; - - struct ggml_object * objects_begin; - struct ggml_object * objects_end; - - struct ggml_scratch scratch; - struct ggml_scratch scratch_save; -}; - -struct ggml_context_container { - bool used; - - struct ggml_context context; -}; - // // NUMA support // @@ -11815,9 +11844,101 @@ static bool ggml_compute_forward_mul_mat_use_blas(struct ggml_tensor * dst) { } #endif +static void ggml_compute_forward_mul_mat_one_chunk( + const struct ggml_compute_params * params, + struct ggml_tensor * dst, + const int64_t num_rows_per_vec_dot, + const int64_t ir0_start, + const int64_t ir0_end, + const int64_t ir1_start, + const int64_t ir1_end) { + + const struct ggml_tensor * src0 = dst->src[0]; + const struct ggml_tensor * src1 = dst->src[1]; + + GGML_TENSOR_BINARY_OP_LOCALS + + const enum ggml_type type = src0->type; + + const bool src1_cont = ggml_is_contiguous(src1); + + ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot; + enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type; + + // broadcast factors + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + //printf("ir0_start = %6lld, ir0_end = %6lld, ir1_start = %6lld, ir1_end = %6lld\n", ir0_start, ir0_end, ir1_start, ir1_end); + + // threads with no work simply yield (not sure if it helps) + if (ir0_start >= ir0_end || ir1_start >= ir1_end) { + return; + } + + const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + const size_t row_size = ggml_row_size(vec_dot_type, ne10); + + assert(ne12 % ne02 == 0); + assert(ne13 % ne03 == 0); + + // block-tiling attempt + const int64_t blck_0 = 16; + const int64_t blck_1 = 16; + + const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; + + // attempt to reduce false-sharing (does not seem to make a difference) + // 16 * 2, accounting for mmla kernels + float tmp[32]; + + for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) { + for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) { + for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) { + const int64_t i13 = (ir1 / (ne12 * ne1)); + const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1; + const int64_t i11 = (ir1 - i13 * ne12 * ne1 - i12 * ne1); + + // broadcast src0 into src1 + const int64_t i03 = i13 / r3; + const int64_t i02 = i12 / r2; + + const int64_t i1 = i11; + const int64_t i2 = i12; + const int64_t i3 = i13; + + const char * src0_row = (const char*)src0->data + (0 + i02 * nb02 + i03 * nb03); + + // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides + // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using + // the original src1 data pointer, so we should index using the indices directly + // TODO: this is a bit of a hack, we should probably have a better way to handle this + const char * src1_col = (const char*)wdata + + (src1_cont || src1->type != vec_dot_type + ? (i11 + i12 * ne11 + i13 * ne12 * ne11) * row_size + : (i11 * nb11 + i12 * nb12 + i13 * nb13)); + float * dst_col = (float*)((char*)dst->data + (i1 * nb1 + i2 * nb2 + i3 * nb3)); + + //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { + // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); + //} + + for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { + vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot); + } + + for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { + memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float)); + } + } + } + } +} + static void ggml_compute_forward_mul_mat( const struct ggml_compute_params * params, - struct ggml_tensor * dst) { + struct ggml_tensor * dst, + struct ggml_compute_state * state) { const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; @@ -11832,9 +11953,6 @@ static void ggml_compute_forward_mul_mat( const enum ggml_type type = src0->type; - const bool src1_cont = ggml_is_contiguous(src1); - - ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot; enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type; ggml_from_float_t const from_float_to_vec_dot = type_traits[vec_dot_type].from_float; int64_t const vec_dot_num_rows = type_traits[type].nrows; @@ -11855,8 +11973,10 @@ static void ggml_compute_forward_mul_mat( GGML_ASSERT(nb2 <= nb3); // broadcast factors - const int64_t r2 = ne12/ne02; - const int64_t r3 = ne13/ne03; + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + UNUSED(r2); + UNUSED(r3); // nb01 >= nb00 - src0 is not transposed // compute by src0 rows @@ -11938,6 +12058,8 @@ static void ggml_compute_forward_mul_mat( #endif #if GGML_USE_LLAMAFILE + const bool src1_cont = ggml_is_contiguous(src1); + if (src1_cont) { for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) @@ -11963,6 +12085,8 @@ UseGgmlGemm1:; if (ith != 0) { return; } + // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start. + atomic_store(&state->shared->current_chunk, nth); if (src1->type != vec_dot_type) { char * wdata = params->wdata; const size_t row_size = ggml_row_size(vec_dot_type, ne10); @@ -11987,11 +12111,11 @@ UseGgmlGemm1:; return; } - const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; - const size_t row_size = ggml_row_size(vec_dot_type, ne10); - #if GGML_USE_LLAMAFILE if (src1->type != vec_dot_type) { + const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata; + const size_t row_size = ggml_row_size(vec_dot_type, ne10); + for (int64_t i13 = 0; i13 < ne13; i13++) for (int64_t i12 = 0; i12 < ne12; i12++) if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type), @@ -12012,98 +12136,87 @@ UseGgmlGemm1:; UseGgmlGemm2:; #endif - const int64_t nr0 = ne01; // src0 rows - const int64_t nr1 = ne1*ne12*ne13; // src1 rows +#ifdef GGML_PERF + int chunks_executed = 0; + UNUSED(chunks_executed); +#endif - //printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1); + // This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers) + const int64_t nr0 = ne0; - // distribute the thread work across the inner or outer loop based on which one is larger - - const int64_t nth0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows - const int64_t nth1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows - - const int64_t ith0 = ith % nth0; - const int64_t ith1 = ith / nth0; - - const int64_t dr0 = (nr0 + nth0 - 1)/nth0; - const int64_t dr1 = (nr1 + nth1 - 1)/nth1; - - const int64_t ir010 = dr0*ith0; - const int64_t ir011 = MIN(ir010 + dr0, nr0); - - const int64_t ir110 = dr1*ith1; - const int64_t ir111 = MIN(ir110 + dr1, nr1); - - //printf("ir010 = %6lld, ir011 = %6lld, ir110 = %6lld, ir111 = %6lld\n", ir010, ir011, ir110, ir111); - - // threads with no work simply yield (not sure if it helps) - if (ir010 >= ir011 || ir110 >= ir111) { - sched_yield(); - return; - } - - assert(ne12 % ne02 == 0); - assert(ne13 % ne03 == 0); - - // block-tiling attempt - const int64_t blck_0 = 16; - const int64_t blck_1 = 16; + // This is the size of the rest of the dimensions of the result + const int64_t nr1 = ne1 * ne2 * ne3; // dot kernels can handle 1 row and col at a time, but mmla kernels can process 2 rows and cols - int64_t nrc = vec_dot_num_rows; + int64_t num_rows_per_vec_dot = vec_dot_num_rows; // TODO: currently the mmla kernels support only even numbered rows/cols. // this check can be removed once they are extended to support odd numbered rows/cols too if ((nr0 % 2 != 0) || (ne11 % 2 != 0)) { - nrc = 1; + num_rows_per_vec_dot = 1; } - const size_t src1_col_stride = src1_cont || src1->type != vec_dot_type ? row_size : nb11; + // Now select a reasonable chunk size. + int chunk_size = 16; - // attempt to reduce false-sharing (does not seem to make a difference) - // 16 * 2, accounting for mmla kernels - float tmp[32]; + // We need to step up the size if it's small + if (nr0 == 1 || nr1 == 1) { + chunk_size = 64; + } - for (int64_t iir1 = ir110; iir1 < ir111; iir1 += blck_1) { - for (int64_t iir0 = ir010; iir0 < ir011; iir0 += blck_0) { - for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ir1 += nrc) { - const int64_t i13 = (ir1/(ne12*ne1)); - const int64_t i12 = (ir1 - i13*ne12*ne1)/ne1; - const int64_t i11 = (ir1 - i13*ne12*ne1 - i12*ne1); + // distribute the work across the inner or outer loop based on which one is larger + // The number of chunks in the 0/1 dim. + // CEIL(nr0/chunk_size) + int64_t nchunk0 = (nr0 + chunk_size - 1) / chunk_size; + int64_t nchunk1 = (nr1 + chunk_size - 1) / chunk_size; - // broadcast src0 into src1 - const int64_t i03 = i13/r3; - const int64_t i02 = i12/r2; + // If the chunking is poor for the number of threads on this setup, scrap the whole plan. Re-chunk it by thread. + // Also, chunking by thread was measured to have perform better on NUMA systems. See https://github.com/ggerganov/llama.cpp/pull/6915 + // In theory, chunking should be just as useful on NUMA and non NUMA systems, but testing disagreed with that. + if (nchunk0 * nchunk1 < nth * 4 || ggml_is_numa()) { + // distribute the thread work across the inner or outer loop based on which one is larger + nchunk0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows + nchunk1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows + } - const int64_t i1 = i11; - const int64_t i2 = i12; - const int64_t i3 = i13; + // The number of elements in each chunk + const int64_t dr0 = (nr0 + nchunk0 - 1) / nchunk0; + const int64_t dr1 = (nr1 + nchunk1 - 1) / nchunk1; - const char * src0_row = (const char *) src0->data + (0 + i02*nb02 + i03*nb03); + //if (ith == 0) + // printf("MUL_MAT = [%d, %d, %d, %d] x [%d, %d, %d, %d] = %d x %d = %d. Fp Ops/Ch %d\n", ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, nchunk0, nchunk1, nchunk0 * nchunk1, ne00 * nr0 * nr1 / nchunk0 / nchunk1); - // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides - // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using - // the original src1 data pointer, so we should index using the indices directly - // TODO: this is a bit of a hack, we should probably have a better way to handle this - const char * src1_col = (const char *) wdata + - (src1_cont || src1->type != vec_dot_type - ? (i11 + i12*ne11 + i13*ne12*ne11)*row_size - : (i11*nb11 + i12*nb12 + i13*nb13)); - float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)); + // The first chunk comes from our thread_id, the rest will get auto-assigned. + int current_chunk = ith; - //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) { - // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col); - //} + while (current_chunk < nchunk0 * nchunk1) { + const int64_t ith0 = current_chunk % nchunk0; + const int64_t ith1 = current_chunk / nchunk0; - for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ir0 += nrc) { - vec_dot(ne00, &tmp[ir0 - iir0], (nrc>1 ? 16 : 0), src0_row + ir0*nb01, (nrc>1 ? nb01 : 0), src1_col, (nrc>1 ? src1_col_stride : 0), nrc); - } + const int64_t ir0_start = dr0 * ith0; + const int64_t ir0_end = MIN(ir0_start + dr0, nr0); - for (int cn = 0; cn < nrc; ++cn) { - memcpy(&dst_col[iir0 + cn*nb1/nb0], tmp + (cn*16), (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float)); - } - } + const int64_t ir1_start = dr1 * ith1; + const int64_t ir1_end = MIN(ir1_start + dr1, nr1); + + ggml_compute_forward_mul_mat_one_chunk(params, dst, num_rows_per_vec_dot, ir0_start, ir0_end, ir1_start, ir1_end); + +#ifdef GGML_PERF + chunks_executed++; +#endif + + if (nth >= nchunk0 * nchunk1) { + break; } + + current_chunk = atomic_fetch_add(&state->shared->current_chunk, 1); } + +#ifdef GGML_PERF + // These numbers are useful when trying to measure how well the threading scheduling works. + //int64_t workSize = (ne01 * ne11 * ne12 * ne13 * ne00) / nchunk0 / nchunk1; + //float time = (ggml_perf_time_us() - t0); + //printf("MUL_MAT = %f ms, [%d, %d, %d, %d] x [%d, %d, %d, %d] = %I64u, %f ops/usec in %d chunks.\n", time / 1000.0, ne00, ne01, ne02, ne03, ne10, ne11, ne12, ne13, workSize, (float)workSize/time, chunks_executed); +#endif } // ggml_compute_forward_mul_mat_id @@ -17358,7 +17471,7 @@ static void ggml_compute_forward_cross_entropy_loss_back( ///////////////////////////////// -static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor) { +static void ggml_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor, struct ggml_compute_state * state) { GGML_ASSERT(params); if (tensor->op == GGML_OP_NONE || ggml_is_empty(tensor)) { @@ -17456,7 +17569,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm } break; case GGML_OP_MUL_MAT: { - ggml_compute_forward_mul_mat(params, tensor); + ggml_compute_forward_mul_mat(params, tensor, state); } break; case GGML_OP_MUL_MAT_ID: { @@ -19072,8 +19185,6 @@ typedef int ggml_lock_t; #define GGML_LOCK_INITIALIZER 0 -typedef pthread_t ggml_thread_t; - #define ggml_thread_create pthread_create #define ggml_thread_join pthread_join @@ -19099,8 +19210,6 @@ typedef int ggml_lock_t; #define GGML_LOCK_INITIALIZER 0 -typedef pthread_t ggml_thread_t; - #define ggml_thread_create pthread_create #define ggml_thread_join pthread_join @@ -19180,31 +19289,6 @@ static void set_numa_thread_affinity(int thread_n) { UNUSED(thread_n); } static void clear_numa_thread_affinity(void) {} #endif -struct ggml_compute_state_shared { - const struct ggml_cgraph * cgraph; - const struct ggml_cplan * cplan; - - int64_t perf_node_start_cycles; - int64_t perf_node_start_time_us; - - const int n_threads; - - // synchronization primitives - atomic_int n_active; // num active threads - atomic_int node_n; // active graph node - atomic_int node_task; // active graph node task phase - - ggml_abort_callback abort_callback; // abort ggml_graph_compute when true - void * abort_callback_data; -}; - -struct ggml_compute_state { - ggml_thread_t thrd; - int ith; - struct ggml_compute_state_shared * shared; - enum ggml_status ec; -}; - static void ggml_graph_compute_perf_stats_node(struct ggml_tensor * node, const struct ggml_compute_state_shared * st) { int64_t cycles_cur = ggml_perf_cycles() - st->perf_node_start_cycles; int64_t time_us_cur = ggml_perf_time_us() - st->perf_node_start_time_us; @@ -19477,6 +19561,10 @@ static void ggml_graph_compute_thread_sync_node(int * node_n, struct ggml_comput * node_n = atomic_load(&state->shared->node_n); if (* node_n != last_node_n) break; +#if defined(__SSE3__) + // Tell the processor we're spinning. It's a processor hint for spinlocks. + _mm_pause(); +#endif } } @@ -19491,6 +19579,10 @@ static void ggml_graph_compute_thread_sync_task(int * task_phase, struct ggml_co * task_phase = atomic_load(&state->shared->node_task); if (* task_phase != last_task_phase) break; +#if defined(__SSE3__) + // Tell the processor we're spinning. It's a processor hint for spinlocks. + _mm_pause(); +#endif } } @@ -19530,7 +19622,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { struct ggml_tensor * node = cgraph->nodes[node_n]; if (GGML_OP_HAS_FINALIZE[node->op]) { params.nth = ggml_get_n_tasks(node, n_threads, state->shared->n_threads); - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } ggml_graph_compute_perf_stats_node(node, state->shared); } @@ -19550,17 +19642,17 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { /* INIT */ if (GGML_OP_HAS_INIT[node->op]) { params.type = GGML_TASK_TYPE_INIT; - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } // TODO: maybe push node_n to the atomic but if other threads see n_tasks is 1, // they do something more efficient than spinning (?) params.type = GGML_TASK_TYPE_COMPUTE; - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); if (GGML_OP_HAS_FINALIZE[node->op]) { params.type = GGML_TASK_TYPE_FINALIZE; - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } ggml_graph_compute_perf_stats_node(node, state->shared); @@ -19599,7 +19691,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { if (state->ith < n_tasks) { if (GGML_OP_HAS_INIT[node->op]) { - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } } @@ -19620,7 +19712,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) { if (state->ith < n_tasks) { params.type = GGML_TASK_TYPE_COMPUTE; - ggml_compute_forward(¶ms, node); + ggml_compute_forward(¶ms, node, state); } if (atomic_fetch_sub(&state->shared->n_active, 1) == 1) { @@ -19871,6 +19963,7 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl /*.node_task =*/ GGML_TASK_TYPE_FINALIZE, /*.abort_callback =*/ NULL, /*.abort_callback_data =*/ NULL, + /*.current_chunk; =*/ 0, }; struct ggml_compute_state * workers = alloca(sizeof(struct ggml_compute_state)*n_threads); From 8f7080bf48828b538bc9387c3d150bbd4fb4cf2d Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Wed, 15 May 2024 23:41:03 +0200 Subject: [PATCH 034/137] readme : remove stray double quote (#7310) Signed-off-by: Daniel Bevenius --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f2f8df64..ecbe802df 100644 --- a/README.md +++ b/README.md @@ -532,7 +532,7 @@ Building the program with BLAS support may lead to some performance improvements cmake -B build -DLLAMA_HIPBLAS=ON -DAMDGPU_TARGETS=gfx1030 -DCMAKE_BUILD_TYPE=Release \ && cmake --build build --config Release -- -j 16 ``` - On Linux it is also possible to use unified memory architecture (UMA) to share main memory between the CPU and integrated GPU by setting `-DLLAMA_HIP_UMA=ON"`. + On Linux it is also possible to use unified memory architecture (UMA) to share main memory between the CPU and integrated GPU by setting `-DLLAMA_HIP_UMA=ON`. However, this hurts performance for non-integrated GPUs (but enables working with integrated GPUs). - Using `make` (example for target gfx1030, build with 16 CPU threads): From 13ad16af1231ab2d245d35df3295bcfa23de1305 Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Wed, 15 May 2024 19:47:36 -0700 Subject: [PATCH 035/137] Add support for properly optimized Windows ARM64 builds with LLVM and MSVC (#7191) * logging: add proper checks for clang to avoid errors and warnings with VA_ARGS * build: add CMake Presets and toolchian files for Windows ARM64 * matmul-int8: enable matmul-int8 with MSVC and fix Clang warnings * ci: add support for optimized Windows ARM64 builds with MSVC and LLVM * matmul-int8: fixed typos in q8_0_q8_0 matmuls Co-authored-by: Georgi Gerganov * matmul-int8: remove unnecessary casts in q8_0_q8_0 --------- Co-authored-by: Georgi Gerganov --- .github/workflows/build.yml | 61 ++++++++++++++++++---------------- CMakeLists.txt | 5 +++ CMakePresets.json | 45 +++++++++++++++++++++++++ cmake/arm64-windows-llvm.cmake | 16 +++++++++ cmake/arm64-windows-msvc.cmake | 6 ++++ common/log.h | 10 +++--- ggml-quants.c | 53 +++++++++++++++-------------- 7 files changed, 138 insertions(+), 58 deletions(-) create mode 100644 CMakePresets.json create mode 100644 cmake/arm64-windows-llvm.cmake create mode 100644 cmake/arm64-windows-msvc.cmake diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ac0e5f6e..2d2fea4a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -693,26 +693,28 @@ jobs: strategy: matrix: include: - - build: 'rpc' + - build: 'rpc-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_RPC=ON -DBUILD_SHARED_LIBS=ON' - - build: 'noavx' + - build: 'noavx-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DBUILD_SHARED_LIBS=ON' - - build: 'avx2' + - build: 'avx2-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' - - build: 'avx' + - build: 'avx-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX2=OFF -DBUILD_SHARED_LIBS=ON' - - build: 'avx512' + - build: 'avx512-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_AVX512=ON -DBUILD_SHARED_LIBS=ON' - - build: 'clblast' + - build: 'clblast-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_CLBLAST=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/clblast"' - - build: 'openblas' + - build: 'openblas-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DBUILD_SHARED_LIBS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"' - - build: 'kompute' + - build: 'kompute-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON' - - build: 'vulkan' + - build: 'vulkan-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=ON' - - build: 'arm64' - defines: '-A ARM64 -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' + - build: 'llvm-arm64' + defines: '-G Ninja -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' + - build: 'msvc-arm64' + defines: '-G Ninja -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc.cmake -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' steps: - name: Clone @@ -723,13 +725,13 @@ jobs: - name: Clone Kompute submodule id: clone_kompute - if: ${{ matrix.build == 'kompute' }} + if: ${{ matrix.build == 'kompute-x64' }} run: | git submodule update --init kompute - name: Download OpenCL SDK id: get_opencl - if: ${{ matrix.build == 'clblast' }} + if: ${{ matrix.build == 'clblast-x64' }} run: | curl.exe -o $env:RUNNER_TEMP/opencl.zip -L "https://github.com/KhronosGroup/OpenCL-SDK/releases/download/v${env:OPENCL_VERSION}/OpenCL-SDK-v${env:OPENCL_VERSION}-Win-x64.zip" mkdir $env:RUNNER_TEMP/opencl @@ -737,7 +739,7 @@ jobs: - name: Download CLBlast id: get_clblast - if: ${{ matrix.build == 'clblast' }} + if: ${{ matrix.build == 'clblast-x64' }} run: | curl.exe -o $env:RUNNER_TEMP/clblast.7z -L "https://github.com/CNugteren/CLBlast/releases/download/${env:CLBLAST_VERSION}/CLBlast-${env:CLBLAST_VERSION}-windows-x64.7z" curl.exe -o $env:RUNNER_TEMP/CLBlast.LICENSE.txt -L "https://github.com/CNugteren/CLBlast/raw/${env:CLBLAST_VERSION}/LICENSE" @@ -750,7 +752,7 @@ jobs: - name: Download OpenBLAS id: get_openblas - if: ${{ matrix.build == 'openblas' }} + if: ${{ matrix.build == 'openblas-x64' }} run: | curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip" curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE" @@ -763,38 +765,41 @@ jobs: - name: Install Vulkan SDK id: get_vulkan - if: ${{ matrix.build == 'kompute' || matrix.build == 'vulkan' }} + if: ${{ matrix.build == 'kompute-x64' || matrix.build == 'vulkan-x64' }} run: | curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/VulkanSDK-${env:VULKAN_VERSION}-Installer.exe" & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}" Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin" + - name: Install Ninja + id: install_ninja + run: | + choco install ninja + - name: Build id: cmake_build run: | - mkdir build - cd build - cmake .. ${{ matrix.defines }} - cmake --build . --config Release -j ${env:NUMBER_OF_PROCESSORS} + cmake -S . -B build ${{ matrix.defines }} + cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS} - name: Add clblast.dll id: add_clblast_dll - if: ${{ matrix.build == 'clblast' }} + if: ${{ matrix.build == 'clblast-x64' }} run: | cp $env:RUNNER_TEMP/clblast/lib/clblast.dll ./build/bin/Release cp $env:RUNNER_TEMP/CLBlast.LICENSE.txt ./build/bin/Release/CLBlast-${env:CLBLAST_VERSION}.txt - name: Add libopenblas.dll id: add_libopenblas_dll - if: ${{ matrix.build == 'openblas' }} + if: ${{ matrix.build == 'openblas-x64' }} run: | cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt - name: Check AVX512F support id: check_avx512f - if: ${{ matrix.build == 'avx512' }} + if: ${{ matrix.build == 'avx512-x64' }} continue-on-error: true run: | cd build @@ -808,14 +813,14 @@ jobs: - name: Test id: cmake_test # not all machines have native AVX-512 - if: ${{ matrix.build != 'arm64' && matrix.build != 'clblast' && matrix.build != 'kompute' && matrix.build != 'vulkan' && (matrix.build != 'avx512' || env.HAS_AVX512F == '1') }} + if: ${{ matrix.build != 'msvc-arm64' && matrix.build != 'llvm-arm64' && matrix.build != 'clblast-x64' && matrix.build != 'kompute-x64' && matrix.build != 'vulkan-x64' && (matrix.build != 'avx512-x64' || env.HAS_AVX512F == '1') }} run: | cd build ctest -L main -C Release --verbose --timeout 900 - name: Test (Intel SDE) id: cmake_test_sde - if: ${{ matrix.build == 'avx512' && env.HAS_AVX512F == '0' }} # use Intel SDE for AVX-512 emulation + if: ${{ matrix.build == 'avx512-x64' && env.HAS_AVX512F == '0' }} # use Intel SDE for AVX-512 emulation run: | curl.exe -o $env:RUNNER_TEMP/sde.tar.xz -L "https://downloadmirror.intel.com/813591/sde-external-${env:SDE_VERSION}-win.tar.xz" # for some weird reason windows tar doesn't like sde tar.xz @@ -843,14 +848,14 @@ jobs: if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} run: | Copy-Item LICENSE .\build\bin\Release\llama.cpp.txt - 7z a llama-${{ steps.tag.outputs.name }}-bin-win-${{ matrix.build }}-x64.zip .\build\bin\Release\* + 7z a llama-${{ steps.tag.outputs.name }}-bin-win-${{ matrix.build }}.zip .\build\bin\Release\* - name: Upload artifacts if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }} uses: actions/upload-artifact@v4 with: - path: llama-${{ steps.tag.outputs.name }}-bin-win-${{ matrix.build }}-x64.zip - name: llama-bin-win-${{ matrix.build }}-x64.zip + path: llama-${{ steps.tag.outputs.name }}-bin-win-${{ matrix.build }}.zip + name: llama-bin-win-${{ matrix.build }}.zip windows-latest-cmake-cuda: runs-on: windows-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index feb6f39d0..8ab6a45a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1007,6 +1007,11 @@ if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STR if (GGML_COMPILER_SUPPORT_DOTPROD) add_compile_definitions(__ARM_FEATURE_DOTPROD) endif () + check_cxx_source_compiles("#include \nint main() { int8x16_t _a, _b; int32x4_t _s = vmlaq_f32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_MATMUL_INT8) + if (GGML_COMPILER_SUPPORT_MATMUL_INT8) + add_compile_definitions(__ARM_FEATURE_MATMUL_INT8) + endif () + check_cxx_source_compiles("#include \nint main() { float16_t _a; float16x8_t _s = vdupq_n_f16(_a); return 0; }" GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC) if (GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC) add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 000000000..ad1af7ecc --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,45 @@ +{ + "version": 4, + "configurePresets": [ + { + "name": "base", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/build-${presetName}", + "cacheVariables": { + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "CMAKE_INSTALL_RPATH": "$ORIGIN;$ORIGIN/.." + } + }, + + { "name": "debug", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" } }, + { "name": "release", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } }, + { "name": "static", "hidden": true, "cacheVariables": { "LLAMA_STATIC": "ON" } }, + + { + "name": "arm64-windows-msvc", "hidden": true, + "architecture": { "value": "arm64", "strategy": "external" }, + "toolset": { "value": "host=x86_64", "strategy": "external" }, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-msvc.cmake" + } + }, + + { + "name": "arm64-windows-llvm", "hidden": true, + "architecture": { "value": "arm64", "strategy": "external" }, + "toolset": { "value": "host=x86_64", "strategy": "external" }, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/cmake/arm64-windows-llvm.cmake" + } + }, + + { "name": "arm64-windows-llvm-debug" , "inherits": [ "base", "arm64-windows-llvm", "debug" ] }, + { "name": "arm64-windows-llvm-release", "inherits": [ "base", "arm64-windows-llvm", "release" ] }, + { "name": "arm64-windows-llvm+static-release", "inherits": [ "base", "arm64-windows-llvm", "release", "static" ] }, + + { "name": "arm64-windows-msvc-debug" , "inherits": [ "base", "arm64-windows-msvc", "debug" ] }, + { "name": "arm64-windows-msvc-release", "inherits": [ "base", "arm64-windows-msvc", "release" ] }, + { "name": "arm64-windows-msvc+static-release", "inherits": [ "base", "arm64-windows-msvc", "release", "static" ] } + ] +} diff --git a/cmake/arm64-windows-llvm.cmake b/cmake/arm64-windows-llvm.cmake new file mode 100644 index 000000000..46fba6514 --- /dev/null +++ b/cmake/arm64-windows-llvm.cmake @@ -0,0 +1,16 @@ +set( CMAKE_SYSTEM_NAME Windows ) +set( CMAKE_SYSTEM_PROCESSOR arm64 ) + +set( target arm64-pc-windows-msvc ) + +set( CMAKE_C_COMPILER clang ) +set( CMAKE_CXX_COMPILER clang++ ) + +set( CMAKE_C_COMPILER_TARGET ${target} ) +set( CMAKE_CXX_COMPILER_TARGET ${target} ) + +set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast" ) +set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) + +set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) +set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/cmake/arm64-windows-msvc.cmake b/cmake/arm64-windows-msvc.cmake new file mode 100644 index 000000000..c77631420 --- /dev/null +++ b/cmake/arm64-windows-msvc.cmake @@ -0,0 +1,6 @@ +set( CMAKE_SYSTEM_NAME Windows ) +set( CMAKE_SYSTEM_PROCESSOR arm64 ) + +set( target arm64-pc-windows-msvc ) +set( CMAKE_C_COMPILER_TARGET ${target} ) +set( CMAKE_CXX_COMPILER_TARGET ${target} ) diff --git a/common/log.h b/common/log.h index 6934c57b2..09fa63c26 100644 --- a/common/log.h +++ b/common/log.h @@ -211,7 +211,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std:: #define LOG_FLF_VAL , __FILE__, __LINE__, __FUNCTION__ #else #define LOG_FLF_FMT "[%24s:%5ld][%24s] " - #define LOG_FLF_VAL , __FILE__, __LINE__, __FUNCTION__ + #define LOG_FLF_VAL , __FILE__, (long)__LINE__, __FUNCTION__ #endif #else #define LOG_FLF_FMT "%s" @@ -224,7 +224,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std:: #define LOG_TEE_FLF_VAL , __FILE__, __LINE__, __FUNCTION__ #else #define LOG_TEE_FLF_FMT "[%24s:%5ld][%24s] " - #define LOG_TEE_FLF_VAL , __FILE__, __LINE__, __FUNCTION__ + #define LOG_TEE_FLF_VAL , __FILE__, (long)__LINE__, __FUNCTION__ #endif #else #define LOG_TEE_FLF_FMT "%s" @@ -294,7 +294,7 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std:: // Main LOG macro. // behaves like printf, and supports arguments the exact same way. // -#ifndef _MSC_VER +#if !defined(_MSC_VER) || defined(__clang__) #define LOG(...) LOG_IMPL(__VA_ARGS__, "") #else #define LOG(str, ...) LOG_IMPL("%s" str, "", ##__VA_ARGS__, "") @@ -308,14 +308,14 @@ inline std::string log_filename_generator_impl(LogTriState multilog, const std:: // Secondary target can be changed just like LOG_TARGET // by defining LOG_TEE_TARGET // -#ifndef _MSC_VER +#if !defined(_MSC_VER) || defined(__clang__) #define LOG_TEE(...) LOG_TEE_IMPL(__VA_ARGS__, "") #else #define LOG_TEE(str, ...) LOG_TEE_IMPL("%s" str, "", ##__VA_ARGS__, "") #endif // LOG macro variants with auto endline. -#ifndef _MSC_VER +#if !defined(_MSC_VER) || defined(__clang__) #define LOGLN(...) LOG_IMPL(__VA_ARGS__, "\n") #define LOG_TEELN(...) LOG_TEE_IMPL(__VA_ARGS__, "\n") #else diff --git a/ggml-quants.c b/ggml-quants.c index 9e62a3f32..f13599f6b 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3487,10 +3487,9 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r #if defined(__ARM_FEATURE_MATMUL_INT8) if (nrc == 2) { const block_q4_0 * restrict vx0 = vx; - const block_q4_0 * restrict vx1 = vx + bx; - + const block_q4_0 * restrict vx1 = (const block_q4_0 *) ((const uint8_t*)vx + bx); const block_q8_0 * restrict vy0 = vy; - const block_q8_0 * restrict vy1 = vy + by; + const block_q8_0 * restrict vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by); float32x4_t sumv0 = vdupq_n_f32(0.0f); @@ -3524,10 +3523,12 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r const int8x16_t y1_l = vld1q_s8(b_y1->qs); const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16); - float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d), - GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d), - GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d), - GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)}; + float32_t _scale[4] = { GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)}; + + float32x4_t scale = vld1q_f32(_scale); int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); @@ -3894,9 +3895,9 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r #if defined(__ARM_FEATURE_MATMUL_INT8) if (nrc == 2) { const block_q4_1 * restrict vx0 = vx; - const block_q4_1 * restrict vx1 = vx + bx; + const block_q4_1 * restrict vx1 = (const block_q4_1 *) ((const uint8_t*)vx + bx); const block_q8_1 * restrict vy0 = vy; - const block_q8_1 * restrict vy1 = vy + by; + const block_q8_1 * restrict vy1 = (const block_q8_1 *) ((const uint8_t*)vy + by); float32x4_t sumv0 = vdupq_n_f32(0.0f); float32x4_t summs0 = vdupq_n_f32(0.0f); @@ -3907,11 +3908,11 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const block_q8_1 * restrict b_y0 = &vy0[i]; const block_q8_1 * restrict b_y1 = &vy1[i]; - float32x4_t summs_t = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s), - GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s), - GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s), - GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)}; - summs0 += summs_t; + float32_t summs_t[4] = {GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y0->s), + GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y0->s), + GGML_FP16_TO_FP32(b_x0->m) * GGML_FP16_TO_FP32(b_y1->s), + GGML_FP16_TO_FP32(b_x1->m) * GGML_FP16_TO_FP32(b_y1->s)}; + summs0 = vaddq_f32(summs0, vld1q_f32(summs_t)); const uint8x16_t m4b = vdupq_n_u8(0x0F); @@ -3931,10 +3932,11 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16); // mmla into int32x4_t - float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d, - GGML_FP16_TO_FP32(b_x0->d)*b_y1->d, - GGML_FP16_TO_FP32(b_x1->d)*b_y0->d, - GGML_FP16_TO_FP32(b_x1->d)*b_y1->d}; + float32_t _scale[4] = {GGML_FP16_TO_FP32(b_x0->d)*b_y0->d, + GGML_FP16_TO_FP32(b_x0->d)*b_y1->d, + GGML_FP16_TO_FP32(b_x1->d)*b_y0->d, + GGML_FP16_TO_FP32(b_x1->d)*b_y1->d}; + float32x4_t scale = vld1q_f32(_scale); int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); @@ -3953,7 +3955,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r float32x4_t sumv1 = vextq_f32(sumv0, sumv0, 2); float32x4_t sumv2 = vzip1q_f32(sumv0, sumv1); - sumv2 = sumv2 + summs0; + sumv2 = vaddq_f32(sumv2, summs0); vst1_f32(s, vget_low_f32(sumv2)); vst1_f32(s + bs, vget_high_f32(sumv2)); @@ -4837,9 +4839,9 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r #if defined(__ARM_FEATURE_MATMUL_INT8) if (nrc == 2) { const block_q8_0 * restrict vx0 = vx; - const block_q8_0 * restrict vx1 = vx + bx; + const block_q8_0 * restrict vx1 = (const block_q8_0 *) ((const uint8_t*)vx + bx); const block_q8_0 * restrict vy0 = vy; - const block_q8_0 * restrict vy1 = vy + by; + const block_q8_0 * restrict vy1 = (const block_q8_0 *) ((const uint8_t*)vy + by); float32x4_t sumv0 = vdupq_n_f32(0.0f); @@ -4861,10 +4863,11 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r const int8x16_t y1_l = vld1q_s8(b_y1->qs); const int8x16_t y1_h = vld1q_s8(b_y1->qs + 16); - float32x4_t scale = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d), - GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d), - GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d), - GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)}; + float32_t _scale[4] = {GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x0->d)*GGML_FP16_TO_FP32(b_y1->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y0->d), + GGML_FP16_TO_FP32(b_x1->d)*GGML_FP16_TO_FP32(b_y1->d)}; + float32x4_t scale = vld1q_f32(_scale); int8x16_t l0 = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); int8x16_t l1 = vreinterpretq_s8_s64(vzip2q_s64(vreinterpretq_s64_s8(x0_l), vreinterpretq_s64_s8(x1_l))); From 172b78210aae0e54d3668c5de14200efab9fac23 Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Wed, 15 May 2024 22:36:43 -0700 Subject: [PATCH 036/137] ci: fix bin/Release path for windows-arm64 builds (#7317) Switch to Ninja Multi-Config CMake generator to resurect bin/Release path that broke artifact packaging in CI. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2d2fea4a2..0742443c6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -712,9 +712,9 @@ jobs: - build: 'vulkan-x64' defines: '-DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DLLAMA_VULKAN=ON -DBUILD_SHARED_LIBS=ON' - build: 'llvm-arm64' - defines: '-G Ninja -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' - build: 'msvc-arm64' - defines: '-G Ninja -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc.cmake -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' + defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc.cmake -DLLAMA_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON' steps: - name: Clone From ad52d5c259344888b06fd5acd3344c663dd0621d Mon Sep 17 00:00:00 2001 From: Vaibhav Srivastav Date: Thu, 16 May 2024 07:38:43 +0200 Subject: [PATCH 037/137] doc: add references to hugging face GGUF-my-repo quantisation web tool. (#7288) * chore: add references to the quantisation space. * fix grammer lol. * Update README.md Co-authored-by: Julien Chaumond * Update README.md Co-authored-by: Georgi Gerganov --------- Co-authored-by: Julien Chaumond Co-authored-by: Georgi Gerganov --- README.md | 3 +++ examples/quantize/README.md | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ecbe802df..5d6217d13 100644 --- a/README.md +++ b/README.md @@ -712,6 +712,9 @@ Building the program with BLAS support may lead to some performance improvements ### Prepare and Quantize +> [!NOTE] +> You can use the [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space on Hugging Face to quantise your model weights without any setup too. It is synced from `llama.cpp` main every 6 hours. + To obtain the official LLaMA 2 weights please see the Obtaining and using the Facebook LLaMA 2 model section. There is also a large selection of pre-quantized `gguf` models available on Hugging Face. Note: `convert.py` does not support LLaMA 3, you can use `convert-hf-to-gguf.py` with LLaMA 3 downloaded from Hugging Face. diff --git a/examples/quantize/README.md b/examples/quantize/README.md index 8a10365c0..b78ece4e7 100644 --- a/examples/quantize/README.md +++ b/examples/quantize/README.md @@ -1,6 +1,8 @@ # quantize -TODO +You can also use the [GGUF-my-repo](https://huggingface.co/spaces/ggml-org/gguf-my-repo) space on Hugging Face to build your own quants without any setup. + +Note: It is synced from llama.cpp `main` every 6 hours. ## Llama 2 7B From 0350f5815218c483fb3026a86adc44a115481625 Mon Sep 17 00:00:00 2001 From: Herman Semenov Date: Thu, 16 May 2024 06:14:24 +0000 Subject: [PATCH 038/137] grammar, json, llama: replace push on emplace if it possible (#7273) --- common/grammar-parser.cpp | 2 +- common/json-schema-to-grammar.cpp | 12 ++++++------ llama.cpp | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/common/grammar-parser.cpp b/common/grammar-parser.cpp index fecb7cd71..b5bc7d49b 100644 --- a/common/grammar-parser.cpp +++ b/common/grammar-parser.cpp @@ -26,7 +26,7 @@ namespace grammar_parser { static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) { uint32_t next_id = static_cast(state.symbol_ids.size()); - auto result = state.symbol_ids.insert(std::make_pair(std::string(src, len), next_id)); + auto result = state.symbol_ids.emplace(std::string(src, len), next_id); return result.first->second; } diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index 0f8f1b1d4..9a71f5d8d 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -272,7 +272,7 @@ private: if (literal.empty()) { return false; } - ret.push_back(std::make_pair(literal, true)); + ret.emplace_back(literal, true); literal.clear(); return true; }; @@ -298,7 +298,7 @@ private: while (i < length) { char c = sub_pattern[i]; if (c == '.') { - seq.push_back(std::make_pair(get_dot(), false)); + seq.emplace_back(get_dot(), false); i++; } else if (c == '(') { i++; @@ -307,7 +307,7 @@ private: _warnings.push_back("Unsupported pattern syntax"); } } - seq.push_back(std::make_pair("(" + to_rule(transform()) + ")", false)); + seq.emplace_back("(" + to_rule(transform()) + ")", false); } else if (c == ')') { i++; if (start > 0 && sub_pattern[start - 1] != '(') { @@ -331,9 +331,9 @@ private: } square_brackets += ']'; i++; - seq.push_back(std::make_pair(square_brackets, false)); + seq.emplace_back(square_brackets, false); } else if (c == '|') { - seq.push_back(std::make_pair("|", false)); + seq.emplace_back("|", false); i++; } else if (c == '*' || c == '+' || c == '?') { seq.back() = std::make_pair(to_rule(seq.back()) + c, false); @@ -417,7 +417,7 @@ private: } } if (!literal.empty()) { - seq.push_back(std::make_pair(literal, true)); + seq.emplace_back(literal, true); } } } diff --git a/llama.cpp b/llama.cpp index 7d26966e4..0ef756a52 100644 --- a/llama.cpp +++ b/llama.cpp @@ -17015,13 +17015,13 @@ static size_t llama_state_seq_get_data_internal(struct llama_context * ctx, llam } else { if (cell_range_begin != kv_self.size) { - cell_ranges.push_back({ cell_range_begin, i }); + cell_ranges.emplace_back(cell_range_begin, i); cell_range_begin = kv_self.size; } } } if (cell_range_begin != kv_self.size) { - cell_ranges.push_back({ cell_range_begin, kv_self.size }); + cell_ranges.emplace_back(cell_range_begin, kv_self.size); } // DEBUG CHECK: Sum of cell counts in ranges should equal the total cell count From dda64fc17c97820ea9489eb0cc9ae8b8fdce4926 Mon Sep 17 00:00:00 2001 From: Jared Van Bortel Date: Thu, 16 May 2024 02:15:23 -0400 Subject: [PATCH 039/137] convert : get general.name from model dir, not its parent (#5615) Co-authored-by: Brian --- convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert.py b/convert.py index e2e642351..da1247957 100755 --- a/convert.py +++ b/convert.py @@ -1109,7 +1109,7 @@ class OutputFile: if metadata is not None and metadata.name is not None: name = metadata.name elif params.path_model is not None: - name = str(params.path_model.parent).split("/")[-1] + name = params.path_model.name elif params.n_ctx == 4096: # Heuristic detection of LLaMA v2 model name = "LLaMA v2" From 3b3963c55c8332e33533c44b2aa882b0e45f8292 Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Wed, 15 May 2024 15:29:07 +0300 Subject: [PATCH 040/137] rpc : add command line arg for specifying backend memory ref: #7293 --- examples/rpc/README.md | 4 +-- examples/rpc/rpc-server.cpp | 68 +++++++++++++++++++++++++++++++------ ggml-rpc.cpp | 2 +- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/examples/rpc/README.md b/examples/rpc/README.md index 325d0abc4..eeec71a8e 100644 --- a/examples/rpc/README.md +++ b/examples/rpc/README.md @@ -42,7 +42,7 @@ cmake --build . --config Release Then, start the `rpc-server` with the backend: ```bash -$ bin/rpc-server 0.0.0.0 50052 +$ bin/rpc-server -p 50052 create_backend: using CUDA backend ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no ggml_cuda_init: CUDA_USE_TENSOR_CORES: yes @@ -53,7 +53,7 @@ Starting RPC server on 0.0.0.0:50052 When using the CUDA backend, you can specify the device with the `CUDA_VISIBLE_DEVICES` environment variable, e.g.: ```bash -$ CUDA_VISIBLE_DEVICES=0 bin/rpc-server 0.0.0.0 50052 +$ CUDA_VISIBLE_DEVICES=0 bin/rpc-server -p 50052 ``` This way you can run multiple `rpc-server` instances on the same host, each with a different CUDA device. diff --git a/examples/rpc/rpc-server.cpp b/examples/rpc/rpc-server.cpp index 496af8496..021185b83 100644 --- a/examples/rpc/rpc-server.cpp +++ b/examples/rpc/rpc-server.cpp @@ -10,6 +10,52 @@ #include #include +struct rpc_server_params { + std::string host = "0.0.0.0"; + int port = 50052; + size_t backend_mem = 0; +}; + +static void print_usage(int /*argc*/, char ** argv, rpc_server_params params) { + fprintf(stderr, "Usage: %s [options]\n\n", argv[0]); + fprintf(stderr, "options:\n"); + fprintf(stderr, " -h, --help show this help message and exit\n"); + fprintf(stderr, " -H HOST, --host HOST host to bind to (default: %s)\n", params.host.c_str()); + fprintf(stderr, " -p PORT, --port PORT port to bind to (default: %d)\n", params.port); + fprintf(stderr, " -m MEM, --mem MEM backend memory size (in MB)\n"); + fprintf(stderr, "\n"); +} + +static bool rpc_server_params_parse(int argc, char ** argv, rpc_server_params & params) { + std::string arg; + for (int i = 1; i < argc; i++) { + arg = argv[i]; + if (arg == "-H" || arg == "--host") { + if (++i >= argc) { + return false; + } + params.host = argv[i]; + } else if (arg == "-p" || arg == "--port") { + if (++i >= argc) { + return false; + } + params.port = std::stoi(argv[i]); + if (params.port <= 0 || params.port > 65535) { + return false; + } + } else if (arg == "-m" || arg == "--mem") { + if (++i >= argc) { + return false; + } + params.backend_mem = std::stoul(argv[i]) * 1024 * 1024; + } else if (arg == "-h" || arg == "--help") { + print_usage(argc, argv, params); + exit(0); + } + } + return true; +} + static ggml_backend_t create_backend() { ggml_backend_t backend = NULL; #ifdef GGML_USE_CUDA @@ -45,14 +91,9 @@ static void get_backend_memory(size_t * free_mem, size_t * total_mem) { } int main(int argc, char * argv[]) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - const char * host = argv[1]; - int port = std::stoi(argv[2]); - if (port <= 0 || port > 65535) { - fprintf(stderr, "Invalid port number: %d\n", port); + rpc_server_params params; + if (!rpc_server_params_parse(argc, argv, params)) { + fprintf(stderr, "Invalid parameters\n"); return 1; } ggml_backend_t backend = create_backend(); @@ -60,10 +101,15 @@ int main(int argc, char * argv[]) { fprintf(stderr, "Failed to create backend\n"); return 1; } - printf("Starting RPC server on %s:%d\n", host, port); + std::string endpoint = params.host + ":" + std::to_string(params.port); size_t free_mem, total_mem; - get_backend_memory(&free_mem, &total_mem); - std::string endpoint = std::string(host) + ":" + std::to_string(port); + if (params.backend_mem > 0) { + free_mem = params.backend_mem; + total_mem = params.backend_mem; + } else { + get_backend_memory(&free_mem, &total_mem); + } + printf("Starting RPC server on %s, backend memory: %zu MB\n", endpoint.c_str(), free_mem / (1024 * 1024)); start_rpc_server(backend, endpoint.c_str(), free_mem, total_mem); ggml_backend_free(backend); return 0; diff --git a/ggml-rpc.cpp b/ggml-rpc.cpp index efeacb297..ba392009f 100644 --- a/ggml-rpc.cpp +++ b/ggml-rpc.cpp @@ -28,7 +28,7 @@ #define UNUSED GGML_UNUSED -#define GGML_DEBUG 1 +#define GGML_DEBUG 0 #if (GGML_DEBUG >= 1) #define GGML_PRINT_DEBUG(...) printf(__VA_ARGS__) #else From 9afdffe70ebf3166d429b4434783bb0b7f97bdeb Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Wed, 15 May 2024 16:04:40 +0300 Subject: [PATCH 041/137] rpc : get available mem for the CPU backend This can be overridden with the -m command line option ref: #7293 --- examples/rpc/rpc-server.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/examples/rpc/rpc-server.cpp b/examples/rpc/rpc-server.cpp index 021185b83..41f377376 100644 --- a/examples/rpc/rpc-server.cpp +++ b/examples/rpc/rpc-server.cpp @@ -7,6 +7,11 @@ #endif #include "ggml-rpc.h" +#ifdef _WIN32 +# include +#else +# include +#endif #include #include @@ -84,9 +89,18 @@ static void get_backend_memory(size_t * free_mem, size_t * total_mem) { #ifdef GGML_USE_CUDA ggml_backend_cuda_get_device_memory(0, free_mem, total_mem); #else - // TODO: implement for other backends - *free_mem = 1; - *total_mem = 1; + #ifdef _WIN32 + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + GlobalMemoryStatusEx(&status); + *total_mem = status.ullTotalPhys; + *free_mem = status.ullAvailPhys; + #else + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGE_SIZE); + *total_mem = pages * page_size; + *free_mem = *total_mem; + #endif #endif } From 24ecb58168dce81646c2ed425690a106591c8c6d Mon Sep 17 00:00:00 2001 From: Pierrick Hymbert Date: Thu, 16 May 2024 20:43:45 +0200 Subject: [PATCH 042/137] Revert "server bench: fix bench not waiting for model load (#7284)" (#7334) This reverts commit 583fd6b000ec9ad1b465b5c98524f4a0ae388077. --- examples/server/bench/bench.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/server/bench/bench.py b/examples/server/bench/bench.py index 25ac29c4c..86c5de101 100644 --- a/examples/server/bench/bench.py +++ b/examples/server/bench/bench.py @@ -293,14 +293,13 @@ def start_server_background(args): def is_server_listening(server_fqdn, server_port): - try: - url = f"{server_fqdn}:{server_port}/health" - if not url.startswith("http://"): - url = f"http://{url}" - result = requests.get(url) - return result.status_code == 200 - except Exception: - return False + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + result = sock.connect_ex((server_fqdn, server_port)) + _is_server_listening = result == 0 + if _is_server_listening: + print(f"server is listening on {server_fqdn}:{server_port}...") + return _is_server_listening + def escape_metric_name(metric_name): return re.sub('[^A-Z0-9]', '_', metric_name.upper()) From 9c4fdcbec8c7fcc428e723b0d8a1cf1f351ba642 Mon Sep 17 00:00:00 2001 From: Leon Knauer Date: Fri, 17 May 2024 02:11:03 +0200 Subject: [PATCH 043/137] [Server] Added --verbose option to README [no ci] (#7335) --- examples/server/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index f6eb6942c..13cbdcd2c 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -17,7 +17,8 @@ The project is under active development, and we are [looking for feedback and co **Command line options:** -- `--threads N`, `-t N`: Set the number of threads to use during generation. Not used if model layers are offloaded to GPU. The server is using batching. This parameter is used only if one token is to be processed on CPU backend. +- `-v`, `--verbose`: Enable verbose server output. When using the `/completion` endpoint, this includes the tokenized prompt, the full request and the full response. +- `-t N`, `--threads N`: Set the number of threads to use during generation. Not used if model layers are offloaded to GPU. The server is using batching. This parameter is used only if one token is to be processed on CPU backend. - `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. Not used if model layers are offloaded to GPU. - `--threads-http N`: Number of threads in the http server pool to process requests. Default: `max(std::thread::hardware_concurrency() - 1, --parallel N + 2)` - `-m FNAME`, `--model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.gguf`). @@ -36,9 +37,7 @@ The project is under active development, and we are [looking for feedback and co - `--numa STRATEGY`: Attempt one of the below optimization strategies that may help on some NUMA systems - `--numa distribute`: Spread execution evenly over all nodes - `--numa isolate`: Only spawn threads on CPUs on the node that execution started on -- `--numa numactl`: Use the CPU map provided by numactl. If run without this previously, it is recommended to drop the system -page cache before using this. See https://github.com/ggerganov/llama.cpp/issues/1437 - +- `--numa numactl`: Use the CPU map provided by numactl. If run without this previously, it is recommended to drop the system page cache before using this. See https://github.com/ggerganov/llama.cpp/issues/1437 - `--numa`: Attempt optimizations that may help on some NUMA systems. - `--lora FNAME`: Apply a LoRA (Low-Rank Adaptation) adapter to the model (implies --no-mmap). This allows you to adapt the pretrained model to specific tasks or domains. - `--lora-base FNAME`: Optional model to use as a base for the layers modified by the LoRA adapter. This flag is used in conjunction with the `--lora` flag, and specifies the base model for the adaptation. From 934266c0e0b2aa9781fdba2deb112c161ff038a9 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Fri, 17 May 2024 02:58:52 -0400 Subject: [PATCH 044/137] ggml : rewrite silu and softmax for cpu (#7154) This change upstreams llamafile's vectorized expf() functions. This lets us compute softmax and silu more accurately than the short[65536] lookup table that GGML previously used to make this operation go faster. We can support aarch64 and sse2+ with the worst case rounding error of 2ulp. It makes make -j8 tests && ./tests/test-backend-ops -o SOFT_MAX -b CPU perf go 1.5x faster for SSE2+FMA, 1.9x faster for AVX2+FMA and 2.1x on AVX512 --- ggml.c | 476 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 283 insertions(+), 193 deletions(-) diff --git a/ggml.c b/ggml.c index 684d84235..55152bce4 100644 --- a/ggml.c +++ b/ggml.c @@ -165,9 +165,6 @@ void ggml_print_backtrace(void) { #define GGML_DEBUG 0 #define GGML_GELU_FP16 #define GGML_GELU_QUICK_FP16 -#define GGML_SILU_FP16 -// #define GGML_CROSS_ENTROPY_EXP_FP16 -// #define GGML_FLASH_ATTN_EXP_FP16 #define GGML_SOFT_MAX_UNROLL 4 #define GGML_VEC_DOT_UNROLL 2 @@ -318,12 +315,6 @@ static ggml_fp16_t ggml_table_gelu_f16[1 << 16]; // precomputed quick gelu table for f16 (128 KB) static ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16]; -// precomputed silu table for f16 (128 KB) -static ggml_fp16_t ggml_table_silu_f16[1 << 16]; - -// precomputed exp table for f16 (128 KB) -static ggml_fp16_t ggml_table_exp_f16[1 << 16]; - // precomputed f32 table for f16 (256 KB) (ggml-impl.h) float ggml_table_f32_f16[1 << 16]; @@ -2085,52 +2076,291 @@ inline static float ggml_silu_f32(float x) { return x/(1.0f + expf(-x)); } -//inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { -// const uint16_t * i16 = (const uint16_t *) x; -// for (int i = 0; i < n; ++i) { -// y[i] = ggml_table_silu_f16[i16[i]]; -// } -//} +#if defined(__ARM_NEON) -#ifdef GGML_SILU_FP16 -inline static void ggml_vec_silu_f32(const int n, float * y, const float * x) { - uint16_t t; - for (int i = 0; i < n; ++i) { - ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]); - memcpy(&t, &fp16, sizeof(uint16_t)); - y[i] = GGML_FP16_TO_FP32(ggml_table_silu_f16[t]); - } +// adapted from arm limited optimized routine +// the maximum error is 1.45358 plus 0.5 ulps +// numbers above 88.38 will flush to infinity +// numbers beneath -103.97 will flush to zero +inline static float32x4_t ggml_v_expf(float32x4_t x) { + const float32x4_t r = vdupq_n_f32(0x1.8p23f); + const float32x4_t z = vfmaq_f32(r, x, vdupq_n_f32(0x1.715476p+0f)); + const float32x4_t n = vsubq_f32(z, r); + const float32x4_t b = vfmsq_f32(vfmsq_f32(x, n, vdupq_n_f32(0x1.62e4p-1f)), n, + vdupq_n_f32(0x1.7f7d1cp-20f)); + const uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_f32(z), 23); + const float32x4_t k = vreinterpretq_f32_u32(vaddq_u32(e, vreinterpretq_u32_f32(vdupq_n_f32(1)))); + const uint32x4_t c = vcagtq_f32(n, vdupq_n_f32(126)); + const float32x4_t u = vmulq_f32(b, b); + const float32x4_t j = vfmaq_f32( + vmulq_f32(vdupq_n_f32(0x1.ffffecp-1f), b), + vfmaq_f32(vfmaq_f32(vdupq_n_f32(0x1.fffdb6p-2f), vdupq_n_f32(0x1.555e66p-3f), b), + vfmaq_f32(vdupq_n_f32(0x1.573e2ep-5f), vdupq_n_f32(0x1.0e4020p-7f), b), u), u); + if (!vpaddd_u64(vreinterpretq_u64_u32(c))) + return vfmaq_f32(k, j, k); + const uint32x4_t d = vandq_u32(vclezq_f32(n), vdupq_n_u32(0x82000000)); + const float32x4_t s1 = vreinterpretq_f32_u32(vaddq_u32(d, vdupq_n_u32(0x7f000000))); + const float32x4_t s2 = vreinterpretq_f32_u32(vsubq_u32(e, d)); + return vbslq_f32(vcagtq_f32(n, vdupq_n_f32(192)), vmulq_f32(s1, s1), + vbslq_f32(c, vmulq_f32(vfmaq_f32(s2, s2, j), s1), vfmaq_f32(k, k, j))); } + +// computes silu x/(1+exp(-x)) in single precision vector +inline static float32x4_t ggml_v_silu(float32x4_t x) { + const float32x4_t one = vdupq_n_f32(1.0f); + const float32x4_t zero = vdupq_n_f32(0.0f); + const float32x4_t neg_x = vsubq_f32(zero, x); + const float32x4_t exp_neg_x = ggml_v_expf(neg_x); + const float32x4_t one_plus_exp_neg_x = vaddq_f32(one, exp_neg_x); + return vdivq_f32(x, one_plus_exp_neg_x); +} + +#elif defined(__AVX512F__) && defined(__AVX512DQ__) + +// adapted from arm limited optimized routine +// the maximum error is 1.45358 plus 0.5 ulps +// numbers above 88.38 will flush to infinity +// numbers beneath -103.97 will flush to zero +inline static __m512 ggml_v_expf(__m512 x) { + const __m512 r = _mm512_set1_ps(0x1.8p23f); + const __m512 z = _mm512_fmadd_ps(x, _mm512_set1_ps(0x1.715476p+0f), r); + const __m512 n = _mm512_sub_ps(z, r); + const __m512 b = _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.7f7d1cp-20f), + _mm512_fnmadd_ps(n, _mm512_set1_ps(0x1.62e4p-1f), x)); + const __m512i e = _mm512_slli_epi32(_mm512_castps_si512(z), 23); + const __m512 k = _mm512_castsi512_ps(_mm512_add_epi32(e, _mm512_castps_si512(_mm512_set1_ps(1)))); + const __mmask16 c = _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(126), _CMP_GT_OQ); + const __m512 u = _mm512_mul_ps(b, b); + const __m512 j = _mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_fmadd_ps(_mm512_set1_ps(0x1.0e4020p-7f), b, + _mm512_set1_ps(0x1.573e2ep-5f)), u, + _mm512_fmadd_ps(_mm512_set1_ps(0x1.555e66p-3f), b, + _mm512_set1_ps(0x1.fffdb6p-2f))), + u, _mm512_mul_ps(_mm512_set1_ps(0x1.ffffecp-1f), b)); + if (_mm512_kortestz(c, c)) + return _mm512_fmadd_ps(j, k, k); + const __m512i g = _mm512_and_si512( + _mm512_movm_epi32(_mm512_cmp_ps_mask(n, _mm512_setzero_ps(), _CMP_LE_OQ)), + _mm512_set1_epi32(0x82000000u)); + const __m512 s1 = + _mm512_castsi512_ps(_mm512_add_epi32(g, _mm512_set1_epi32(0x7f000000u))); + const __m512 s2 = _mm512_castsi512_ps(_mm512_sub_epi32(e, g)); + const __mmask16 d = + _mm512_cmp_ps_mask(_mm512_abs_ps(n), _mm512_set1_ps(192), _CMP_GT_OQ); + return _mm512_mask_blend_ps( + d, _mm512_mask_blend_ps( + c, _mm512_fmadd_ps(k, j, k), + _mm512_mul_ps(_mm512_fmadd_ps(s2, j, s2), s1)), + _mm512_mul_ps(s1, s1)); +} + +// computes silu x/(1+exp(-x)) in single precision vector +inline static __m512 ggml_v_silu(__m512 x) { + const __m512 one = _mm512_set1_ps(1); + const __m512 zero = _mm512_setzero_ps(); + const __m512 neg_x = _mm512_sub_ps(zero, x); + const __m512 exp_neg_x = ggml_v_expf(neg_x); + const __m512 one_plus_exp_neg_x = _mm512_add_ps(one, exp_neg_x); + return _mm512_div_ps(x, one_plus_exp_neg_x); +} + +#elif defined(__AVX2__) && defined(__FMA__) + +// adapted from arm limited optimized routine +// the maximum error is 1.45358 plus 0.5 ulps +// numbers above 88.38 will flush to infinity +// numbers beneath -103.97 will flush to zero +inline static __m256 ggml_v_expf(__m256 x) { + const __m256 r = _mm256_set1_ps(0x1.8p23f); + const __m256 z = _mm256_fmadd_ps(x, _mm256_set1_ps(0x1.715476p+0f), r); + const __m256 n = _mm256_sub_ps(z, r); + const __m256 b = _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.7f7d1cp-20f), + _mm256_fnmadd_ps(n, _mm256_set1_ps(0x1.62e4p-1f), x)); + const __m256i e = _mm256_slli_epi32(_mm256_castps_si256(z), 23); + const __m256 k = _mm256_castsi256_ps( + _mm256_add_epi32(e, _mm256_castps_si256(_mm256_set1_ps(1)))); + const __m256i c = _mm256_castps_si256( + _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n), + _mm256_set1_ps(126), _CMP_GT_OQ)); + const __m256 u = _mm256_mul_ps(b, b); + const __m256 j = _mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_fmadd_ps(_mm256_set1_ps(0x1.0e4020p-7f), b, + _mm256_set1_ps(0x1.573e2ep-5f)), u, + _mm256_fmadd_ps(_mm256_set1_ps(0x1.555e66p-3f), b, + _mm256_set1_ps(0x1.fffdb6p-2f))), + u, _mm256_mul_ps(_mm256_set1_ps(0x1.ffffecp-1f), b)); + if (!_mm256_movemask_ps(_mm256_castsi256_ps(c))) + return _mm256_fmadd_ps(j, k, k); + const __m256i g = _mm256_and_si256( + _mm256_castps_si256(_mm256_cmp_ps(n, _mm256_setzero_ps(), _CMP_LE_OQ)), + _mm256_set1_epi32(0x82000000u)); + const __m256 s1 = + _mm256_castsi256_ps(_mm256_add_epi32(g, _mm256_set1_epi32(0x7f000000u))); + const __m256 s2 = _mm256_castsi256_ps(_mm256_sub_epi32(e, g)); + const __m256i d = _mm256_castps_si256( + _mm256_cmp_ps(_mm256_andnot_ps(_mm256_set1_ps(-0.f), n), + _mm256_set1_ps(192), _CMP_GT_OQ)); + return _mm256_or_ps( + _mm256_and_ps(_mm256_castsi256_ps(d), _mm256_mul_ps(s1, s1)), + _mm256_andnot_ps( + _mm256_castsi256_ps(d), + _mm256_or_ps( + _mm256_and_ps(_mm256_castsi256_ps(c), + _mm256_mul_ps(_mm256_fmadd_ps(s2, j, s2), s1)), + _mm256_andnot_ps(_mm256_castsi256_ps(c), _mm256_fmadd_ps(k, j, k))))); +} + +// computes silu x/(1+exp(-x)) in single precision vector +inline static __m256 ggml_v_silu(__m256 x) { + const __m256 one = _mm256_set1_ps(1); + const __m256 zero = _mm256_setzero_ps(); + const __m256 neg_x = _mm256_sub_ps(zero, x); + const __m256 exp_neg_x = ggml_v_expf(neg_x); + const __m256 one_plus_exp_neg_x = _mm256_add_ps(one, exp_neg_x); + return _mm256_div_ps(x, one_plus_exp_neg_x); +} + +#elif defined(__SSE2__) // __AVX2__ / __ARM_NEON + +#if defined(__FMA__) +#define MADD128(x, y, z) _mm_fmadd_ps(x, y, z) +#define NMADD128(x, y, z) _mm_fnmadd_ps(x, y, z) #else -inline static void ggml_vec_silu_f32(const int n, float * y, const float * x) { - for (int i = 0; i < n; ++i) { +#define MADD128(x, y, z) _mm_add_ps(_mm_mul_ps(x, y), z) +#define NMADD128(x, y, z) _mm_sub_ps(z, _mm_mul_ps(x, y)) +#endif + +// adapted from arm limited optimized routine +// the maximum error is 1.45358 plus 0.5 ulps +// numbers above 88.38 will flush to infinity +// numbers beneath -103.97 will flush to zero +inline static __m128 ggml_v_expf(__m128 x) { + const __m128 r = _mm_set1_ps(0x1.8p23f); + const __m128 z = MADD128(x, _mm_set1_ps(0x1.715476p+0f), r); + const __m128 n = _mm_sub_ps(z, r); + const __m128 b = + NMADD128(n, _mm_set1_ps(0x1.7f7d1cp-20f), NMADD128(n, _mm_set1_ps(0x1.62e4p-1f), x)); + const __m128i e = _mm_slli_epi32(_mm_castps_si128(z), 23); + const __m128 k = _mm_castsi128_ps(_mm_add_epi32(e, _mm_castps_si128(_mm_set1_ps(1)))); + const __m128i c = + _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(126))); + const __m128 u = _mm_mul_ps(b, b); + const __m128 j = + MADD128(MADD128(MADD128(_mm_set1_ps(0x1.0e4020p-7f), b, _mm_set1_ps(0x1.573e2ep-5f)), u, + MADD128(_mm_set1_ps(0x1.555e66p-3f), b, _mm_set1_ps(0x1.fffdb6p-2f))), + u, _mm_mul_ps(_mm_set1_ps(0x1.ffffecp-1f), b)); + if (!_mm_movemask_epi8(c)) + return MADD128(j, k, k); + const __m128i g = _mm_and_si128(_mm_castps_si128(_mm_cmple_ps(n, _mm_setzero_ps())), + _mm_set1_epi32(0x82000000u)); + const __m128 s1 = _mm_castsi128_ps(_mm_add_epi32(g, _mm_set1_epi32(0x7f000000u))); + const __m128 s2 = _mm_castsi128_ps(_mm_sub_epi32(e, g)); + const __m128i d = + _mm_castps_si128(_mm_cmpgt_ps(_mm_andnot_ps(_mm_set1_ps(-0.f), n), _mm_set1_ps(192))); + return _mm_or_ps( + _mm_and_ps(_mm_castsi128_ps(d), _mm_mul_ps(s1, s1)), + _mm_andnot_ps(_mm_castsi128_ps(d), + _mm_or_ps(_mm_and_ps(_mm_castsi128_ps(c), _mm_mul_ps(MADD128(s2, j, s2), s1)), + _mm_andnot_ps(_mm_castsi128_ps(c), MADD128(k, j, k))))); +} + +// computes silu x/(1+exp(-x)) in single precision vector +inline static __m128 ggml_v_silu(__m128 x) { + const __m128 one = _mm_set1_ps(1); + const __m128 zero = _mm_setzero_ps(); + const __m128 neg_x = _mm_sub_ps(zero, x); + const __m128 exp_neg_x = ggml_v_expf(neg_x); + const __m128 one_plus_exp_neg_x = _mm_add_ps(one, exp_neg_x); + return _mm_div_ps(x, one_plus_exp_neg_x); +} + +#endif // __ARM_NEON / __AVX2__ / __SSE2__ + +static void ggml_vec_silu_f32(const int n, float * y, const float * x) { + int i = 0; +#if defined(__AVX512F__) && defined(__AVX512DQ__) + for (; i + 15 < n; i += 16) { + _mm512_storeu_ps(y + i, ggml_v_silu(_mm512_loadu_ps(x + i))); + } +#elif defined(__AVX2__) && defined(__FMA__) + for (; i + 7 < n; i += 8) { + _mm256_storeu_ps(y + i, ggml_v_silu(_mm256_loadu_ps(x + i))); + } +#elif defined(__SSE2__) + for (; i + 3 < n; i += 4) { + _mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i))); + } +#elif defined(__ARM_NEON) + for (; i + 3 < n; i += 4) { + vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i))); + } +#endif + for (; i < n; ++i) { y[i] = ggml_silu_f32(x[i]); } } + +static ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max) { + int i = 0; + ggml_float sum = 0; +#if defined(__AVX512F__) && defined(__AVX512DQ__) + for (; i + 15 < n; i += 16) { + __m512 val = ggml_v_expf(_mm512_sub_ps(_mm512_loadu_ps(x + i), + _mm512_set1_ps(max))); + _mm512_storeu_ps(y + i, val); + sum += (ggml_float)_mm512_reduce_add_ps(val); + } +#elif defined(__AVX2__) && defined(__FMA__) + for (; i + 7 < n; i += 8) { + __m256 val = ggml_v_expf(_mm256_sub_ps(_mm256_loadu_ps(x + i), + _mm256_set1_ps(max))); + _mm256_storeu_ps(y + i, val); + __m128 val2 = _mm_add_ps(_mm256_extractf128_ps(val, 1), + _mm256_castps256_ps128(val)); + val2 = _mm_add_ps(val2, _mm_movehl_ps(val2, val2)); + val2 = _mm_add_ss(val2, _mm_movehdup_ps(val2)); + sum += (ggml_float)_mm_cvtss_f32(val2); + } +#elif defined(__SSE2__) + for (; i + 3 < n; i += 4) { + __m128 val = ggml_v_expf(_mm_sub_ps(_mm_loadu_ps(x + i), + _mm_set1_ps(max))); + _mm_storeu_ps(y + i, val); +#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) + val = _mm_add_ps(val, _mm_movehl_ps(val, val)); + val = _mm_add_ss(val, _mm_movehdup_ps(val)); +#else + __m128 tmp = _mm_shuffle_ps(val, val, _MM_SHUFFLE(2, 3, 0, 1)); + val = _mm_add_ps(val, tmp); + tmp = _mm_movehl_ps(tmp, val); + val = _mm_add_ss(val, tmp); #endif + sum += (ggml_float)_mm_cvtss_f32(val); + } +#elif defined(__ARM_NEON) + for (; i + 3 < n; i += 4) { + float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i), + vdupq_n_f32(max))); + vst1q_f32(y + i, val); + sum += (ggml_float)vaddvq_f32(val); + } +#endif + for (; i < n; ++i) { + float val = expf(x[i] - max); + sum += (ggml_float)val; + y[i] = val; + } + return sum; +} inline static float ggml_silu_backward_f32(float x, float dy) { const float s = 1.0f/(1.0f + expf(-x)); return dy*s*(1.0f + x*(1.0f - s)); } -#ifdef GGML_SILU_FP16 -inline static void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) { - for (int i = 0; i < n; ++i) { - // we did not use x[i] to compute forward silu but its f16 equivalent - // take derivative at f16 of x[i]: - ggml_fp16_t fp16 = GGML_FP32_TO_FP16(x[i]); - float usedx = GGML_FP16_TO_FP32(fp16); - dx[i] = ggml_silu_backward_f32(usedx, dy[i]); - } -} -#else inline static void ggml_vec_silu_backward_f32(const int n, float * dx, const float * x, const float * dy) { for (int i = 0; i < n; ++i) { dx[i] = ggml_silu_backward_f32(x[i], dy[i]); } } -#endif inline static void ggml_vec_sum_f32(const int n, float * s, const float * x) { #ifndef GGML_USE_ACCELERATE @@ -2922,8 +3152,6 @@ struct ggml_context * ggml_init(struct ggml_init_params params) { float f = ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(u.fp16); ggml_table_gelu_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_f32(f)); ggml_table_gelu_quick_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_quick_f32(f)); - ggml_table_silu_f16[i] = GGML_FP32_TO_FP16(ggml_silu_f32(f)); - ggml_table_exp_f16[i] = GGML_FP32_TO_FP16(expf(f)); } const uint64_t t_end = ggml_time_us(); UNUSED(t_end); @@ -13600,22 +13828,7 @@ static void ggml_compute_forward_soft_max_f32( float max = -INFINITY; ggml_vec_max_f32(nc, &max, wp); - ggml_float sum = 0.0; - - uint16_t scvt; - for (int i = 0; i < nc; i++) { - if (wp[i] == -INFINITY) { - dp[i] = 0.0f; - } else { - // const float val = (wp[i] == -INFINITY) ? 0.0 : exp(wp[i] - max); - ggml_fp16_t s = GGML_FP32_TO_FP16(wp[i] - max); - memcpy(&scvt, &s, sizeof(scvt)); - const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt]); - sum += (ggml_float)val; - dp[i] = val; - } - } - + ggml_float sum = ggml_vec_soft_max_f32(nc, dp, wp, max); assert(sum > 0.0); sum = 1.0/sum; @@ -15374,37 +15587,7 @@ static void ggml_compute_forward_flash_attn_f32( vvexpf(S, S, &Mup); ggml_vec_sum_f32(Mup, &sum, S); #else - uint16_t scvt[GGML_SOFT_MAX_UNROLL]; UNUSED(scvt); - ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 }; - - for (int i = 0; i < Mup; i += GGML_SOFT_MAX_UNROLL) { - if (i >= masked_begin) { - break; - } - float * SS = S + i; - - for (int j = 0; j < GGML_SOFT_MAX_UNROLL; ++j) { - if (i + j >= masked_begin) { - break; - } else if (SS[j] == -INFINITY) { - SS[j] = 0.0f; - } else { -#ifndef GGML_FLASH_ATTN_EXP_FP16 - const float val = expf(SS[j] - max); -#else - ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max); - memcpy(&scvt[j], &s, sizeof(uint16_t)); - const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt[j]]); -#endif - sump[j] += (ggml_float)val; - SS[j] = val; - } - } - } - - for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) { - sum += sump[i]; - } + sum = ggml_vec_soft_max_f32(Mup, S, S, max); #endif } @@ -15586,28 +15769,7 @@ static void ggml_compute_forward_flash_attn_f16( vvexpf(S, S, &Mup); ggml_vec_sum_f32(Mup, &sum, S); #else - uint16_t scvt[GGML_SOFT_MAX_UNROLL]; - ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 }; - - for (int i = 0; i < Mup; i += GGML_SOFT_MAX_UNROLL) { - float * SS = S + i; - - for (int j = 0; j < GGML_SOFT_MAX_UNROLL; ++j) { - if (SS[j] == -INFINITY) { - SS[j] = 0.0f; - } else { - ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max); - memcpy(&scvt[j], &s, sizeof(uint16_t)); - const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt[j]]); - sump[j] += (ggml_float)val; - SS[j] = val; - } - } - } - - for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) { - sum += sump[i]; - } + sum = ggml_vec_soft_max_f32(Mup, S, S, max); #endif } @@ -16234,38 +16396,7 @@ static void ggml_compute_forward_flash_attn_back_f32( vvexpf(SM, SM, &Mup); ggml_vec_sum_f32(Mup, &sum, SM); #else - uint16_t scvt[GGML_SOFT_MAX_UNROLL]; UNUSED(scvt); - ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 }; - - for (int i = 0; i < Mup; i += GGML_SOFT_MAX_UNROLL) { - if (i >= masked_begin) { - break; - } - float * SR = S + i; - float * SW = SM + i; - - for (int j = 0; j < GGML_SOFT_MAX_UNROLL; ++j) { - if (i + j >= masked_begin) { - break; - } else if (SR[j] == -INFINITY) { - SW[j] = 0.0f; - } else { -#ifndef GGML_FLASH_ATTN_EXP_FP16 - const float val = expf(SR[j] - max); -#else - ggml_fp16_t s = GGML_FP32_TO_FP16(SR[j] - max); - memcpy(&scvt[j], &s, sizeof(uint16_t)); - const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt[j]]); -#endif - sump[j] += (ggml_float)val; - SW[j] = val; - } - } - } - - for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) { - sum += sump[i]; - } + sum = ggml_vec_soft_max_f32(Mup, SM, S, max); #endif } @@ -17291,35 +17422,15 @@ static void ggml_compute_forward_cross_entropy_loss_f32( assert(!isnan(s1[i])); } #endif + // soft_max - ggml_float sum = 0.0; - { - float max = -INFINITY; - ggml_vec_max_f32(nc, &max, s0); - - uint16_t scvt; UNUSED(scvt); - for (int i = 0; i < nc; i++) { - if (s0[i] == -INFINITY) { - st[i] = 0.0f; - } else { -#ifndef GGML_CROSS_ENTROPY_EXP_FP16 - const float s = s0[i] - max; - const float val = expf(s); -#else - ggml_fp16_t s = GGML_FP32_TO_FP16(s0[i] - max); - memcpy(&scvt, &s, sizeof(scvt)); - const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt]); -#endif - sum += (ggml_float)val; - st[i] = val; - } - } - - assert(sum > 0.0); - // sum = 1.0/sum; - } - // avoid log(0) by rescaling from [0..1] to [eps..1] + float max = -INFINITY; + ggml_vec_max_f32(nc, &max, s0); + ggml_float sum = ggml_vec_soft_max_f32(nc, st, s0, max); + assert(sum > 0.0); sum = (1.0 - eps) / sum; + + // avoid log(0) by rescaling from [0..1] to [eps..1] ggml_vec_scale_f32(nc, st, sum); ggml_vec_add1_f32(nc, st, st, eps); ggml_vec_log_f32(nc, st, st); @@ -17409,32 +17520,11 @@ static void ggml_compute_forward_cross_entropy_loss_back_f32( #endif // soft_max - ggml_float sum = 0.0; - { - float max = -INFINITY; - ggml_vec_max_f32(nc, &max, s0); - - uint16_t scvt; UNUSED(scvt); - for (int i = 0; i < nc; i++) { - if (s0[i] == -INFINITY) { - ds0[i] = 0.0f; - } else { -#ifndef GGML_CROSS_ENTROPY_EXP_FP16 - const float s = s0[i] - max; - const float val = expf(s); -#else - ggml_fp16_t s = GGML_FP32_TO_FP16(s0[i] - max); - memcpy(&scvt, &s, sizeof(scvt)); - const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt]); -#endif - sum += (ggml_float)val; - ds0[i] = val; - } - } - - assert(sum > 0.0); - sum = (1.0 - eps)/sum; - } + float max = -INFINITY; + ggml_vec_max_f32(nc, &max, s0); + ggml_float sum = ggml_vec_soft_max_f32(nc, ds0, s0, max); + assert(sum > 0.0); + sum = (1.0 - eps) / sum; // grad(src0) = (softmax(src0) - src1) * grad(cross_entropy_loss(src0, src1)) / nr ggml_vec_scale_f32(nc, ds0, sum); From ee94172d33399d2e814ca05c8a3ff8c523ebb093 Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Fri, 17 May 2024 10:00:17 +0300 Subject: [PATCH 045/137] server : add support for the RPC backend (#7305) ref: #7292 --- examples/server/server.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 7e0d068f8..b598076d3 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -2387,6 +2387,7 @@ static void server_print_usage(const char * argv0, const gpt_params & params, co printf(" --lora-base FNAME optional model to use as a base for the layers modified by the LoRA adapter\n"); printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str()); printf(" --port PORT port to listen (default (default: %d)\n", sparams.port); + printf(" --rpc SERVERS comma separated list of RPC servers\n"); printf(" --path PUBLIC_PATH path from which to serve static files (default: disabled)\n"); printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n"); printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n"); @@ -2439,6 +2440,12 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams, break; } sparams.port = std::stoi(argv[i]); + } else if (arg == "--rpc") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.rpc_servers = argv[i]; } else if (arg == "--host") { if (++i >= argc) { invalid_param = true; From e18bc6aaf3b547890609ed254ee5248e720e5840 Mon Sep 17 00:00:00 2001 From: amd-lalithnc Date: Fri, 17 May 2024 12:31:58 +0530 Subject: [PATCH 046/137] convert : fix Qwen/Qwen-7b conversion (#7308) --- convert-hf-to-gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index cd875fa4a..2810e1e41 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -526,7 +526,7 @@ class Model: # for this kind of tokenizer, added_vocab is not a subset of vocab, so they need to be combined added_vocab = tokenizer.special_tokens - reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in (vocab | added_vocab).items()} + reverse_vocab = {id_ : encoded_tok for encoded_tok, id_ in {**vocab, **added_vocab}.items()} for i in range(vocab_size): if i not in reverse_vocab: From 359cbe3f46c90ce6f5151005e411b8fb74f8139e Mon Sep 17 00:00:00 2001 From: Herman Semenov Date: Fri, 17 May 2024 07:08:49 +0000 Subject: [PATCH 047/137] ggml-quants, llama : removed excess checks (#7274) --- common/common.cpp | 2 +- ggml-quants.c | 2 +- llama.cpp | 8 ++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 96130ad54..e624fc7f3 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -2553,7 +2553,7 @@ void dump_string_yaml_multiline(FILE * stream, const char * prop_name, const cha size_t pos_start = 0; size_t pos_found = 0; - if (!data_str.empty() && (std::isspace(data_str[0]) || std::isspace(data_str.back()))) { + if (std::isspace(data_str[0]) || std::isspace(data_str.back())) { data_str = std::regex_replace(data_str, std::regex("\n"), "\\n"); data_str = std::regex_replace(data_str, std::regex("\""), "\\\""); data_str = std::regex_replace(data_str, std::regex(R"(\\[^n"])"), R"(\$&)"); diff --git a/ggml-quants.c b/ggml-quants.c index f13599f6b..9b291d522 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1986,7 +1986,7 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restri for (int j = 0; j < QK_K/16; ++j) { if (quant_weights) { - const float * qw = quant_weights ? quant_weights + QK_K * i + 16*j : NULL; + const float * qw = quant_weights + QK_K * i + 16*j; for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j+l]*x[16*j+l]); } else { for (int l = 0; l < 16; ++l) weight[l] = x[16*j+l]*x[16*j+l]; diff --git a/llama.cpp b/llama.cpp index 0ef756a52..daaa138b9 100644 --- a/llama.cpp +++ b/llama.cpp @@ -13904,9 +13904,7 @@ llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_ // Sample the next word X using top-k sampling llama_sample_top_k(nullptr, candidates, int(k), 1); - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; llama_token X = llama_sample_token(ctx, candidates); t_start_sample_us = ggml_time_us(); @@ -13920,9 +13918,7 @@ llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_ // Update mu using the learning rate and error *mu = *mu - eta * e; - if (ctx) { - ctx->t_sample_us += ggml_time_us() - t_start_sample_us; - } + ctx->t_sample_us += ggml_time_us() - t_start_sample_us; return X; } From 29c60d8cddcfd14fa8a6bf023a6c4eb8692c76ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Fri, 17 May 2024 09:59:57 +0200 Subject: [PATCH 048/137] tokenization: add warning for double BOS (#7332) --- llama.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/llama.cpp b/llama.cpp index daaa138b9..c5a1fa0f6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12818,6 +12818,13 @@ static std::vector llama_tokenize_internal(const llama_vocab & } } + if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) { + LLAMA_LOG_WARN( + "%s: Added a BOS token to the prompt as specified by the model but the prompt " + "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. " + "Are you sure this is what you want?\n", __FUNCTION__); + } + if (add_special && vocab.special_add_eos == 1) { GGML_ASSERT(vocab.special_eos_id != -1); output.push_back(vocab.special_eos_id); @@ -12844,6 +12851,13 @@ static std::vector llama_tokenize_internal(const llama_vocab & } } + if (add_special && vocab.special_add_bos != 0 && output.size() >= 2 && output[1] == vocab.special_bos_id) { + LLAMA_LOG_WARN( + "%s: Added a BOS token to the prompt as specified by the model but the prompt " + "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. " + "Are you sure this is what you want?\n", __FUNCTION__); + } + if (add_special && vocab.special_add_eos == 1) { GGML_ASSERT(vocab.special_add_eos != -1); output.push_back(vocab.special_eos_id); From 27b040691cbe45314147c2745e891a38e9c048d4 Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Fri, 17 May 2024 13:24:38 +0200 Subject: [PATCH 049/137] llama : use n_embd_head_v when reshaping kqv (#7327) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * llama : use n_embd_head_v instead of n_embd_head_k when reshaping kqv * llama : use n_embd_v_gqa and n_embd_head_v instead of n_embd_k_gqa and n_embd_head_k when making a view of cached value vectors. --------- Co-authored-by: Stanisław Szymczyk --- llama.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/llama.cpp b/llama.cpp index c5a1fa0f6..e11f0ac4b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6622,6 +6622,7 @@ static struct ggml_tensor * llm_build_kqv( const int64_t n_embd_head_k = hparams.n_embd_head_k; const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(); const int64_t n_embd_head_v = hparams.n_embd_head_v; + const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(); struct ggml_tensor * q = ggml_permute(ctx, q_cur, 0, 2, 1, 3); cb(q, "q", il); @@ -6644,8 +6645,8 @@ static struct ggml_tensor * llm_build_kqv( struct ggml_tensor * v = ggml_view_3d(ctx, kv.v_l[il], n_embd_head_v, n_kv, n_head_kv, - ggml_row_size(kv.v_l[il]->type, n_embd_k_gqa), - ggml_row_size(kv.v_l[il]->type, n_embd_head_k), + ggml_row_size(kv.v_l[il]->type, n_embd_v_gqa), + ggml_row_size(kv.v_l[il]->type, n_embd_head_v), 0); cb(v, "v", il); @@ -6655,7 +6656,7 @@ static struct ggml_tensor * llm_build_kqv( ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); } - cur = ggml_reshape_2d(ctx, cur, n_embd_head_k*n_head, n_tokens); + cur = ggml_reshape_2d(ctx, cur, n_embd_head_v*n_head, n_tokens); } else { struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); cb(kq, "kq", il); @@ -6700,7 +6701,7 @@ static struct ggml_tensor * llm_build_kqv( struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3); cb(kqv_merged, "kqv_merged", il); - cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_k*n_head, n_tokens); + cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_v*n_head, n_tokens); cb(cur, "kqv_merged_cont", il); } From d273c1402b25086fd91aef2467ac13f2e49fa0ea Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Fri, 17 May 2024 15:11:45 +0300 Subject: [PATCH 050/137] py : convert-hf-to-gguf-update improvements (#7340) * convert-hf-to-gguf-update: automate updating * convert-hf-to-gguf-update: improve download * share requests session for performance * create directories only when needed, don't skip downloads when empty directory encountered * be more graceful about errors --- convert-hf-to-gguf-update.py | 87 ++++++++++++++++-------------------- convert-hf-to-gguf.py | 2 + 2 files changed, 41 insertions(+), 48 deletions(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 14aa0c45a..27983fadf 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -20,11 +20,13 @@ # - Update llama.cpp with the new pre-tokenizer if necessary # # TODO: generate tokenizer tests for llama.cpp -# TODO: automate the update of convert-hf-to-gguf.py # import logging import os +import pathlib +import re + import requests import sys import json @@ -35,6 +37,7 @@ from transformers import AutoTokenizer logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger("convert-hf-to-gguf-update") +sess = requests.Session() class TOKENIZER_TYPE(IntEnum): @@ -79,63 +82,44 @@ models = [ {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", }, ] -# make directory "models/tokenizers" if it doesn't exist -if not os.path.exists("models/tokenizers"): - os.makedirs("models/tokenizers") - def download_file_with_auth(url, token, save_path): headers = {"Authorization": f"Bearer {token}"} - response = requests.get(url, headers=headers) - if response.status_code == 200: - with open(save_path, 'wb') as f: - f.write(response.content) - logger.info(f"File {save_path} downloaded successfully") - else: - logger.info(f"Failed to download file. Status code: {response.status_code}") + response = sess.get(url, headers=headers) + response.raise_for_status() + os.makedirs(os.path.dirname(save_path), exist_ok=True) + with open(save_path, 'wb') as f: + f.write(response.content) + logger.info(f"File {save_path} downloaded successfully") -# download the tokenizer models -for model in models: +def download_model(model): name = model["name"] repo = model["repo"] tokt = model["tokt"] - if not os.path.exists(f"models/tokenizers/{name}"): - os.makedirs(f"models/tokenizers/{name}") - else: - logger.info(f"Directory models/tokenizers/{name} already exists - skipping") - continue - - logger.info(f"Downloading {name} to models/tokenizers/{name}") - - url = f"{repo}/raw/main/config.json" - save_path = f"models/tokenizers/{name}/config.json" - download_file_with_auth(url, token, save_path) - - url = f"{repo}/raw/main/tokenizer.json" - save_path = f"models/tokenizers/{name}/tokenizer.json" - download_file_with_auth(url, token, save_path) - - # if downloaded file is less than 1KB, we likely need to download an LFS instead - if os.path.getsize(save_path) < 1024: - # remove the file - os.remove(save_path) - url = f"{repo}/resolve/main/tokenizer.json" - save_path = f"models/tokenizers/{name}/tokenizer.json" - download_file_with_auth(url, token, save_path) + os.makedirs(f"models/tokenizers/{name}", exist_ok=True) + files = ["config.json", "tokenizer.json", "tokenizer_config.json"] if tokt == TOKENIZER_TYPE.SPM: - url = f"{repo}/resolve/main/tokenizer.model" - save_path = f"models/tokenizers/{name}/tokenizer.model" - download_file_with_auth(url, token, save_path) + files.append("tokenizer.model") + + for file in files: + save_path = f"models/tokenizers/{name}/{file}" + if os.path.isfile(save_path): + logger.info(f"{name}: File {save_path} already exists - skipping") + continue + download_file_with_auth(f"{repo}/resolve/main/{file}", token, save_path) + + +for model in models: + try: + download_model(model) + except Exception as e: + logger.error(f"Failed to download model {model['name']}. Error: {e}") - url = f"{repo}/raw/main/tokenizer_config.json" - save_path = f"models/tokenizers/{name}/tokenizer_config.json" - download_file_with_auth(url, token, save_path) # generate the source code for the convert-hf-to-gguf.py:get_vocab_base_pre() function: -# TODO: auto-update convert-hf-to-gguf.py with the generated function src_ifs = "" for model in models: @@ -224,11 +208,18 @@ src_func = f""" return res """ -print(src_func) # noqa: NP100 +convert_py_pth = pathlib.Path("convert-hf-to-gguf.py") +convert_py = convert_py_pth.read_text() +convert_py = re.sub( + r"(# Marker: Start get_vocab_base_pre)(.+?)( +# Marker: End get_vocab_base_pre)", + lambda m: m.group(1) + src_func + m.group(3), + convert_py, + flags=re.DOTALL | re.MULTILINE, +) -logger.info("\n") -logger.info("!!! Copy-paste the function above into convert-hf-to-gguf.py !!!") -logger.info("\n") +convert_py_pth.write_text(convert_py) + +logger.info("+++ convert-hf-to-gguf.py was updated") # generate tests for each tokenizer model diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 2810e1e41..5ba3161c7 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -402,6 +402,7 @@ class Model: # NOTE: this function is generated by convert-hf-to-gguf-update.py # do not modify it manually! # ref: https://github.com/ggerganov/llama.cpp/pull/6920 + # Marker: Start get_vocab_base_pre def get_vocab_base_pre(self, tokenizer) -> str: # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that # is specific for the BPE pre-tokenizer used by the model @@ -489,6 +490,7 @@ class Model: logger.debug(f"chkhsh: {chkhsh}") return res + # Marker: End get_vocab_base_pre def _set_vocab_gpt2(self) -> None: tokens, toktypes, tokpre = self.get_vocab_base() From 51e9d02599336e62948d29f1d6c05addeb921ac2 Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 17 May 2024 22:40:14 +1000 Subject: [PATCH 051/137] Added a single test function script and fix debug-test.sh to be more robust (#7279) * run-single-test.sh: added a single test function script and fix debug-test.sh to be more robust * debug-test.sh: combined execute and gdb test mode via -g flag * debug-test.sh: refactor * debug-test: refactor for clarity * debug-test.sh: comment style changes * debug-test.sh: fix gdb --- docs/debugging-tests.md | 24 +++- scripts/debug-test.sh | 266 ++++++++++++++++++++++++++-------------- 2 files changed, 196 insertions(+), 94 deletions(-) diff --git a/docs/debugging-tests.md b/docs/debugging-tests.md index 51a125e19..18407f688 100644 --- a/docs/debugging-tests.md +++ b/docs/debugging-tests.md @@ -1,6 +1,6 @@ # Debugging Tests Tips -## How to run & debug a specific test without anything else to keep the feedback loop short? +## How to run & execute or debug a specific test without anything else to keep the feedback loop short? There is a script called debug-test.sh in the scripts folder whose parameter takes a REGEX and an optional test number. @@ -10,13 +10,27 @@ For example, running the following command will output an interactive list from It will then build & run in the debugger for you. +To just execute a test and get back a PASS or FAIL message run: + ```bash ./scripts/debug-test.sh test-tokenizer +``` + +To test in GDB use the `-g` flag to enable gdb test mode. + +```bash +./scripts/debug-test.sh -g test-tokenizer # Once in the debugger, i.e. at the chevrons prompt, setting a breakpoint could be as follows: >>> b main ``` +To speed up the testing loop, if you know your test number you can just run it similar to below: + +```bash +./scripts/debug-test.sh test 23 +``` + For further reference use `debug-test.sh -h` to print help.   @@ -41,7 +55,7 @@ cmake -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON .. make -j ``` -#### Step 3.1: Identify Test Command for Debugging +#### Step 3: Find all tests available that matches REGEX The output of this command will give you the command & arguments needed to run GDB. @@ -69,11 +83,13 @@ Labels: main ... ``` -So for test #1 we can tell these two pieces of relevant information: +#### Step 4: Identify Test Command for Debugging + +So for test #1 above we can tell these two pieces of relevant information: * Test Binary: `~/llama.cpp/build-ci-debug/bin/test-tokenizer-0` * Test GGUF Model: `~/llama.cpp/tests/../models/ggml-vocab-llama-spm.gguf` -#### Step 3.2: Run GDB on test command +#### Step 5: Run GDB on test command Based on the ctest 'test command' report above we can then run a gdb session via this command below: diff --git a/scripts/debug-test.sh b/scripts/debug-test.sh index 231a23d69..7b2b601a9 100755 --- a/scripts/debug-test.sh +++ b/scripts/debug-test.sh @@ -1,117 +1,203 @@ #!/bin/bash -test_suite=${1:-} -test_number=${2:-} PROG=${0##*/} build_dir="build-ci-debug" -if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then - echo "Usage: $PROG [OPTION]... (test_number)" - echo "Debug specific ctest program." - echo - echo "Options:" - echo " -h, --help Display this help and exit" - echo - echo "Arguments:" - echo " (Mandatory) Supply one regex to the script to filter tests" - echo " (test_number) (Optional) Test number to run a specific test" - echo - echo "Example:" - echo " $PROG test-tokenizer" - echo " $PROG test-tokenizer 3" - echo - exit 0 -fi +# Print Color Commands +red=$(tput setaf 1) +green=$(tput setaf 2) +yellow=$(tput setaf 3) +blue=$(tput setaf 4) +magenta=$(tput setaf 5) +cyan=$(tput setaf 6) +normal=$(tput sgr0) -# Function to select and debug a test -function select_test() { - test_suite=${1:-test} - test_number=${2:-} - # Sanity Check If Tests Is Detected - printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" - tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1')) - if [ ${#tests[@]} -eq 0 ] - then - echo "No tests avaliable... check your compliation process..." - echo "Exiting." - exit 1 - fi +# Print Help Message +#################### - if [ -z $test_number ] - then - # List out avaliable tests - printf "Which test would you like to debug?\n" - id=0 - for s in "${tests[@]}" - do - echo "Test# ${id}" - echo " $s" - ((id++)) - done +print_full_help() { + cat << EOF +Usage: $PROG [OPTION]... (test_number) +Debug specific ctest program. - # Prompt user which test they wanted to run - printf "\nRun test#? " - read test_number - else - printf "\nUser Already Requested #${test_number}" - fi +Options: + -h, --help display this help and exit + -g run in gdb mode - # Start GDB with the requested test binary and arguments - printf "Debugging(GDB) test: ${tests[test_number]}\n" - # Change IFS (Internal Field Separator) - sIFS=$IFS - IFS=$'\n' +Arguments: + (Mandatory) Supply one regex to the script to filter tests + (test_number) (Optional) Test number to run a specific test - # Get test args - gdb_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) - IFS=$sIFS - printf "Debug arguments: ${gdb_args[test_number]}\n\n" - - # Expand paths if needed - args=() - for x in $(echo ${gdb_args[test_number]} | sed -e 's/"\/\"//') - do - args+=($(echo $x | sed -e 's/.*\/..\//..\//')) - done - - # Execute debugger - echo "gdb args: ${args[@]}" - gdb --args ${args[@]} +Example: + $PROG test-tokenizer + $PROG test-tokenizer 3 +EOF } +abort() { + echo "Error: $1" >&2 + cat << EOF >&2 +Usage: $PROG [OPTION]... (test_number) +Debug specific ctest program. +Refer to --help for full instructions. +EOF + exit 1 +} + + +# Dependency Sanity Check +######################### + +check_dependency() { + command -v "$1" >/dev/null 2>&1 || { + abort "$1 is required but not found. Please install it and try again." + } +} + +check_dependency ctest +check_dependency cmake + + # Step 0: Check the args -if [ -z "$test_suite" ] -then - echo "Usage: $PROG [OPTION]... (test_number)" - echo "Supply one regex to the script to filter tests," - echo "and optionally a test number to run a specific test." - echo "Use --help flag for full instructions" - exit 1 +######################## + +if [ x"$1" = x"-h" ] || [ x"$1" = x"--help" ]; then + print_full_help >&2 + exit 0 fi +# Parse command-line options +gdb_mode=false +while getopts "g" opt; do + case $opt in + g) + gdb_mode=true + echo "gdb_mode Mode Enabled" + ;; + esac +done + +# Shift the option parameters +shift $((OPTIND - 1)) + +# Positionial Argument Processing : +if [ -z "${1}" ]; then + abort "Test regex is required" +else + test_suite=${1:-} +fi + +# Positionial Argument Processing : (test_number) +test_number=${2:-} + + # Step 1: Reset and Setup folder context +######################################## + ## Sanity check that we are actually in a git repo repo_root=$(git rev-parse --show-toplevel) if [ ! -d "$repo_root" ]; then - echo "Error: Not in a Git repository." - exit 1 + abort "Not in a Git repository." fi -## Reset folder to root context of git repo -pushd "$repo_root" || exit 1 +## Reset folder to root context of git repo and Create and enter build directory +pushd "$repo_root" +rm -rf "$build_dir" && mkdir "$build_dir" || abort "Failed to make $build_dir" -## Create and enter build directory -rm -rf "$build_dir" && mkdir "$build_dir" || exit 1 # Step 2: Setup Build Environment and Compile Test Binaries -cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_FATAL_WARNINGS=ON || exit 1 -pushd "$build_dir" && make -j || exit 1 +########################################################### -# Step 3: Debug the Test -select_test "$test_suite" "$test_number" +# Note: test-eval-callback requires -DLLAMA_CURL +cmake -B "./$build_dir" -DCMAKE_BUILD_TYPE=Debug -DLLAMA_CUDA=1 -DLLAMA_CURL=1 || abort "Failed to build enviroment" +pushd "$build_dir" +make -j || abort "Failed to compile" +popd > /dev/null || exit 1 -# Step 4: Return to the directory from which the user ran the command. -popd || exit 1 -popd || exit 1 -popd || exit 1 + +# Step 3: Find all tests available that matches REGEX +#################################################### + +# Ctest Gather Tests +# `-R test-tokenizer` : looks for all the test files named `test-tokenizer*` (R=Regex) +# `-N` : "show-only" disables test execution & shows test commands that you can feed to GDB. +# `-V` : Verbose Mode +printf "\n\nGathering tests that fit REGEX: ${test_suite} ...\n" +pushd "$build_dir" +tests=($(ctest -R ${test_suite} -V -N | grep -E " +Test +#[0-9]+*" | cut -d':' -f2 | awk '{$1=$1};1')) +if [ ${#tests[@]} -eq 0 ]; then + abort "No tests avaliable... check your compliation process..." +fi +popd > /dev/null || exit 1 + + +# Step 4: Identify Test Command for Debugging +############################################# + +# Select test number +if [ -z $test_number ]; then + # List out avaliable tests + printf "Which test would you like to debug?\n" + id=0 + for s in "${tests[@]}" + do + echo "Test# ${id}" + echo " $s" + ((id++)) + done + + # Prompt user which test they wanted to run + printf "\nRun test#? " + read test_number + +else + printf "\nUser Already Requested #${test_number}\n" + +fi + +# Grab all tests commands +pushd "$build_dir" +sIFS=$IFS # Save Initial IFS (Internal Field Separator) +IFS=$'\n' # Change IFS (Internal Field Separator) (So we split ctest output by newline rather than by spaces) +test_args=($(ctest -R ${test_suite} -V -N | grep "Test command" | cut -d':' -f3 | awk '{$1=$1};1' )) # Get test args +IFS=$sIFS # Reset IFS (Internal Field Separator) +popd > /dev/null || exit 1 + +# Grab specific test command +single_test_name="${tests[test_number]}" +single_test_command="${test_args[test_number]}" + + +# Step 5: Execute or GDB Debug +############################## + +printf "${magenta}Running Test #${test_number}: ${single_test_name}${normal}\n" +printf "${cyan}single_test_command: ${single_test_command}${normal}\n" + +if [ "$gdb_mode" = "true" ]; then + # Execute debugger + pushd "$repo_root" || exit 1 + eval "gdb --args ${single_test_command}" + popd > /dev/null || exit 1 + +else + # Execute Test + pushd "$repo_root" || exit 1 + eval "${single_test_command}" + exit_code=$? + popd > /dev/null || exit 1 + + # Print Result + printf "${blue}Ran Test #${test_number}: ${single_test_name}${normal}\n" + printf "${yellow}Command: ${single_test_command}${normal}\n" + if [ $exit_code -eq 0 ]; then + printf "${green}TEST PASS${normal}\n" + else + printf "${red}TEST FAIL${normal}\n" + fi + +fi + +# Return to the directory from which the user ran the command. +popd > /dev/null || exit 1 From f4bd8b3d260bb09491ba63c77ab7012b744362ef Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Fri, 17 May 2024 17:25:44 +0300 Subject: [PATCH 052/137] rpc : set SO_REUSEADDR for the server socket (#7320) ref: #7293 --- examples/rpc/rpc-server.cpp | 4 ++++ ggml-rpc.cpp | 13 +++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/examples/rpc/rpc-server.cpp b/examples/rpc/rpc-server.cpp index 41f377376..7c15d2aa4 100644 --- a/examples/rpc/rpc-server.cpp +++ b/examples/rpc/rpc-server.cpp @@ -56,6 +56,10 @@ static bool rpc_server_params_parse(int argc, char ** argv, rpc_server_params & } else if (arg == "-h" || arg == "--help") { print_usage(argc, argv, params); exit(0); + } else { + fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); + print_usage(argc, argv, params); + exit(0); } } return true; diff --git a/ggml-rpc.cpp b/ggml-rpc.cpp index ba392009f..4a9bfa52d 100644 --- a/ggml-rpc.cpp +++ b/ggml-rpc.cpp @@ -134,7 +134,13 @@ static bool set_no_delay(sockfd_t sockfd) { int flag = 1; // set TCP_NODELAY to disable Nagle's algorithm int ret = setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, (char *)&flag, sizeof(int)); - return ret >= 0; + return ret == 0; +} + +static bool set_reuse_addr(sockfd_t sockfd) { + int flag = 1; + int ret = setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof(int)); + return ret == 0; } static std::shared_ptr socket_connect(const char * host, int port) { @@ -181,7 +187,10 @@ static std::shared_ptr create_server_socket(const char * host, int por if (sock == nullptr) { return nullptr; } - + if (!set_reuse_addr(sockfd)) { + fprintf(stderr, "Failed to set SO_REUSEADDR\n"); + return nullptr; + } struct sockaddr_in serv_addr; serv_addr.sin_family = AF_INET; serv_addr.sin_addr.s_addr = inet_addr(host); From 82ca83db3c8d45df559c03a4225b6eb34808a2db Mon Sep 17 00:00:00 2001 From: Gavin Zhao Date: Fri, 17 May 2024 11:03:03 -0400 Subject: [PATCH 053/137] ROCm: use native CMake HIP support (#5966) Supercedes #4024 and #4813. CMake's native HIP support has become the recommended way to add HIP code into a project (see [here](https://rocm.docs.amd.com/en/docs-6.0.0/conceptual/cmake-packages.html#using-hip-in-cmake)). This PR makes the following changes: 1. The environment variable `HIPCXX` or CMake option `CMAKE_HIP_COMPILER` should be used to specify the HIP compiler. Notably this shouldn't be `hipcc`, but ROCm's clang, which usually resides in `$ROCM_PATH/llvm/bin/clang`. Previously this was control by `CMAKE_C_COMPILER` and `CMAKE_CXX_COMPILER`. Note that since native CMake HIP support is not yet available on Windows, on Windows we fall back to the old behavior. 2. CMake option `CMAKE_HIP_ARCHITECTURES` is used to control the GPU architectures to build for. Previously this was controled by `GPU_TARGETS`. 3. Updated the Nix recipe to account for these new changes. 4. The GPU targets to build against in the Nix recipe is now consistent with the supported GPU targets in nixpkgs. 5. Added CI checks for HIP on both Linux and Windows. On Linux, we test both the new and old behavior. The most important part about this PR is the separation of the HIP compiler and the C/C++ compiler. This allows users to choose a different C/C++ compiler if desired, compared to the current situation where when building for ROCm support, everything must be compiled with ROCm's clang. ~~Makefile is unchanged. Please let me know if we want to be consistent on variables' naming because Makefile still uses `GPU_TARGETS` to control architectures to build for, but I feel like setting `CMAKE_HIP_ARCHITECTURES` is a bit awkward when you're calling `make`.~~ Makefile used `GPU_TARGETS` but the README says to use `AMDGPU_TARGETS`. For consistency with CMake, all usage of `GPU_TARGETS` in Makefile has been updated to `AMDGPU_TARGETS`. Thanks to the suggestion of @jin-eld, to maintain backwards compatibility (and not break too many downstream users' builds), if `CMAKE_CXX_COMPILER` ends with `hipcc`, then we still compile using the original behavior and emit a warning that recommends switching to the new HIP support. Similarly, if `AMDGPU_TARGETS` is set but `CMAKE_HIP_ARCHITECTURES` is not, then we forward `AMDGPU_TARGETS` to `CMAKE_HIP_ARCHITECTURES` to ease the transition to the new HIP support. Signed-off-by: Gavin Zhao --- .devops/nix/package.nix | 16 +++++----- .github/workflows/build.yml | 58 +++++++++++++++++++++++++++++++++++++ CMakeLists.txt | 42 ++++++++++++++++++++++----- Makefile | 6 ++-- README.md | 25 ++++++++++++---- 5 files changed, 122 insertions(+), 25 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 2c0ae4e2a..1c9633cdf 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -227,20 +227,20 @@ effectiveStdenv.mkDerivation ( ) ] ++ optionals useRocm [ - (cmakeFeature "CMAKE_C_COMPILER" "hipcc") - (cmakeFeature "CMAKE_CXX_COMPILER" "hipcc") - - # Build all targets supported by rocBLAS. When updating search for TARGET_LIST_ROCM - # in https://github.com/ROCmSoftwarePlatform/rocBLAS/blob/develop/CMakeLists.txt - # and select the line that matches the current nixpkgs version of rocBLAS. - # Should likely use `rocmPackages.clr.gpuTargets`. - "-DAMDGPU_TARGETS=gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102" + (cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang") + (cmakeFeature "CMAKE_HIP_ARCHITECTURES" (builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets)) ] ++ optionals useMetalKit [ (lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1") (cmakeBool "LLAMA_METAL_EMBED_LIBRARY" (!precompileMetalShaders)) ]; + # Environment variables needed for ROCm + env = optionals useRocm { + ROCM_PATH = "${rocmPackages.clr}"; + HIP_DEVICE_LIB_PATH = "${rocmPackages.rocm-device-libs}/amdgcn/bitcode"; + }; + # TODO(SomeoneSerge): It's better to add proper install targets at the CMake level, # if they haven't been added yet. postInstall = '' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0742443c6..0109cc004 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -392,6 +392,33 @@ jobs: cmake -DLLAMA_VULKAN=ON .. cmake --build . --config Release -j $(nproc) + ubuntu-22-cmake-hip: + runs-on: ubuntu-22.04 + container: rocm/dev-ubuntu-22.04:6.0.2 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev + + - name: Build with native CMake HIP support + id: cmake_build + run: | + cmake -B build -S . -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" -DLLAMA_HIPBLAS=ON + cmake --build build --config Release -j $(nproc) + + - name: Build with legacy HIP support + id: cmake_build_legacy_hip + run: | + cmake -B build2 -S . -DCMAKE_C_COMPILER=hipcc -DCMAKE_CXX_COMPILER=hipcc -DLLAMA_HIPBLAS=ON + cmake --build build2 --config Release -j $(nproc) + ubuntu-22-cmake-sycl: runs-on: ubuntu-22.04 @@ -989,6 +1016,37 @@ jobs: path: llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip name: llama-bin-win-sycl-x64.zip + windows-latest-cmake-hip: + runs-on: windows-latest + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + + - name: Install + id: depends + run: | + $ErrorActionPreference = "Stop" + write-host "Downloading AMD HIP SDK Installer" + Invoke-WebRequest -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-23.Q4-WinSvr2022-For-HIP.exe" -OutFile "${env:RUNNER_TEMP}\rocm-install.exe" + write-host "Installing AMD HIP SDK" + Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -Wait + write-host "Completed AMD HIP SDK installation" + + - name: Verify ROCm + id: verify + run: | + & 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' --version + + - name: Build + id: cmake_build + run: | + $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path) + $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}" + cmake -G "Unix Makefiles" -B build -S . -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" -DLLAMA_HIPBLAS=ON + cmake --build build --config Release + ios-xcode-build: runs-on: macos-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ab6a45a6..990e34b86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -555,16 +555,37 @@ if (LLAMA_VULKAN) endif() if (LLAMA_HIPBLAS) - list(APPEND CMAKE_PREFIX_PATH /opt/rocm) + if ($ENV{ROCM_PATH}) + set(ROCM_PATH $ENV{ROCM_PATH}) + else() + set(ROCM_PATH /opt/rocm) + endif() + list(APPEND CMAKE_PREFIX_PATH ${ROCM_PATH}) - if (NOT ${CMAKE_C_COMPILER_ID} MATCHES "Clang") - message(WARNING "Only LLVM is supported for HIP, hint: CC=/opt/rocm/llvm/bin/clang") + # CMake on Windows doesn't support the HIP language yet + if(WIN32) + set(CXX_IS_HIPCC TRUE) + else() + string(REGEX MATCH "hipcc(\.bat)?$" CXX_IS_HIPCC "${CMAKE_CXX_COMPILER}") endif() - if (NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - message(WARNING "Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++") - endif() + if(CXX_IS_HIPCC) + if(LINUX) + if (NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + message(WARNING "Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++") + endif() + message(WARNING "Setting hipcc as the C++ compiler is legacy behavior." + " Prefer setting the HIP compiler directly. See README for details.") + endif() + else() + # Forward AMDGPU_TARGETS to CMAKE_HIP_ARCHITECTURES. + if(AMDGPU_TARGETS AND NOT CMAKE_HIP_ARCHITECTURES) + set(CMAKE_HIP_ARCHITECTURES ${AMDGPU_ARGETS}) + endif() + cmake_minimum_required(VERSION 3.21) + enable_language(HIP) + endif() find_package(hip REQUIRED) find_package(hipblas REQUIRED) find_package(rocblas REQUIRED) @@ -598,13 +619,18 @@ if (LLAMA_HIPBLAS) add_compile_definitions(GGML_CUDA_MMV_Y=${LLAMA_CUDA_MMV_Y}) add_compile_definitions(K_QUANTS_PER_ITERATION=${LLAMA_CUDA_KQUANTS_ITER}) - set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE CXX) + if (CXX_IS_HIPCC) + set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE CXX) + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} hip::device) + else() + set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE HIP) + endif() if (LLAMA_STATIC) message(FATAL_ERROR "Static linking not supported for HIP/ROCm") endif() - set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} hip::device PUBLIC hip::host roc::rocblas roc::hipblas) + set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} PUBLIC hip::host roc::rocblas roc::hipblas) endif() if (LLAMA_SYCL) diff --git a/Makefile b/Makefile index 3fa56d13a..22d521856 100644 --- a/Makefile +++ b/Makefile @@ -560,10 +560,10 @@ endif # LLAMA_VULKAN ifdef LLAMA_HIPBLAS ifeq ($(wildcard /opt/rocm),) ROCM_PATH ?= /usr - GPU_TARGETS ?= $(shell $(shell which amdgpu-arch)) + AMDGPU_TARGETS ?= $(shell $(shell which amdgpu-arch)) else ROCM_PATH ?= /opt/rocm - GPU_TARGETS ?= $(shell $(ROCM_PATH)/llvm/bin/amdgpu-arch) + AMDGPU_TARGETS ?= $(shell $(ROCM_PATH)/llvm/bin/amdgpu-arch) endif HIPCC ?= $(CCACHE) $(ROCM_PATH)/bin/hipcc LLAMA_CUDA_DMMV_X ?= 32 @@ -575,7 +575,7 @@ ifdef LLAMA_HIP_UMA endif # LLAMA_HIP_UMA MK_LDFLAGS += -L$(ROCM_PATH)/lib -Wl,-rpath=$(ROCM_PATH)/lib MK_LDFLAGS += -lhipblas -lamdhip64 -lrocblas - HIPFLAGS += $(addprefix --offload-arch=,$(GPU_TARGETS)) + HIPFLAGS += $(addprefix --offload-arch=,$(AMDGPU_TARGETS)) HIPFLAGS += -DGGML_CUDA_DMMV_X=$(LLAMA_CUDA_DMMV_X) HIPFLAGS += -DGGML_CUDA_MMV_Y=$(LLAMA_CUDA_MMV_Y) HIPFLAGS += -DK_QUANTS_PER_ITERATION=$(LLAMA_CUDA_KQUANTS_ITER) diff --git a/README.md b/README.md index 5d6217d13..7dd6fc0eb 100644 --- a/README.md +++ b/README.md @@ -528,13 +528,28 @@ Building the program with BLAS support may lead to some performance improvements ``` - Using `CMake` for Linux (assuming a gfx1030-compatible AMD GPU): ```bash - CC=/opt/rocm/llvm/bin/clang CXX=/opt/rocm/llvm/bin/clang++ \ - cmake -B build -DLLAMA_HIPBLAS=ON -DAMDGPU_TARGETS=gfx1030 -DCMAKE_BUILD_TYPE=Release \ + HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \ + cmake -S . -B build -DLLAMA_HIPBLAS=ON -DAMDGPU_TARGETS=gfx1030 -DCMAKE_BUILD_TYPE=Release \ && cmake --build build --config Release -- -j 16 ``` On Linux it is also possible to use unified memory architecture (UMA) to share main memory between the CPU and integrated GPU by setting `-DLLAMA_HIP_UMA=ON`. However, this hurts performance for non-integrated GPUs (but enables working with integrated GPUs). + Note that if you get the following error: + ``` + clang: error: cannot find ROCm device library; provide its path via '--rocm-path' or '--rocm-device-lib-path', or pass '-nogpulib' to build without ROCm device library + ``` + Try searching for a directory under `HIP_PATH` that contains the file + `oclc_abi_version_400.bc`. Then, add the following to the start of the + command: `HIP_DEVICE_LIB_PATH=`, so something + like: + ```bash + HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -p)" \ + HIP_DEVICE_LIB_PATH= \ + cmake -S . -B build -DLLAMA_HIPBLAS=ON -DAMDGPU_TARGETS=gfx1030 -DCMAKE_BUILD_TYPE=Release \ + && cmake --build build -- -j 16 + ``` + - Using `make` (example for target gfx1030, build with 16 CPU threads): ```bash make -j16 LLAMA_HIPBLAS=1 LLAMA_HIP_UMA=1 AMDGPU_TARGETS=gfx1030 @@ -543,10 +558,8 @@ Building the program with BLAS support may lead to some performance improvements - Using `CMake` for Windows (using x64 Native Tools Command Prompt for VS, and assuming a gfx1100-compatible AMD GPU): ```bash set PATH=%HIP_PATH%\bin;%PATH% - mkdir build - cd build - cmake -G Ninja -DAMDGPU_TARGETS=gfx1100 -DLLAMA_HIPBLAS=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release .. - cmake --build . + cmake -S . -B build -G Ninja -DAMDGPU_TARGETS=gfx1100 -DLLAMA_HIPBLAS=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release + cmake --build build ``` Make sure that `AMDGPU_TARGETS` is set to the GPU arch you want to compile for. The above example uses `gfx1100` that corresponds to Radeon RX 7900XTX/XT/GRE. You can find a list of targets [here](https://llvm.org/docs/AMDGPUUsage.html#processors) Find your gpu version string by matching the most significant version information from `rocminfo | grep gfx | head -1 | awk '{print $2}'` with the list of processors, e.g. `gfx1035` maps to `gfx1030`. From 0fc1e820a9900a3dd08ddd3c6abe6604c53b689b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Fri, 17 May 2024 18:54:52 +0200 Subject: [PATCH 054/137] CUDA: faster large batch FA without tensor cores (#7314) --- ggml-cuda/fattn-tile-f16.cu | 395 +++++++++++++++++++++++++++++++++++ ggml-cuda/fattn-tile-f16.cuh | 3 + ggml-cuda/fattn-tile-f32.cu | 393 ++++++++++++++++++++++++++++++++++ ggml-cuda/fattn-tile-f32.cuh | 3 + ggml-cuda/fattn-vec-f16.cu | 9 +- ggml-cuda/fattn-vec-f32.cu | 9 +- ggml-cuda/fattn.cu | 26 ++- 7 files changed, 823 insertions(+), 15 deletions(-) create mode 100644 ggml-cuda/fattn-tile-f16.cu create mode 100644 ggml-cuda/fattn-tile-f16.cuh create mode 100644 ggml-cuda/fattn-tile-f32.cu create mode 100644 ggml-cuda/fattn-tile-f32.cuh diff --git a/ggml-cuda/fattn-tile-f16.cu b/ggml-cuda/fattn-tile-f16.cu new file mode 100644 index 000000000..d2a1077ed --- /dev/null +++ b/ggml-cuda/fattn-tile-f16.cu @@ -0,0 +1,395 @@ +#include "common.cuh" +#include "fattn-common.cuh" +#include "fattn-tile-f16.cuh" + +#define FATTN_KQ_STRIDE_TILE_F16 64 + +template // D == head size +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +__launch_bounds__(nwarps*WARP_SIZE, 1) +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +static __global__ void flash_attn_tile_ext_f16( + const char * __restrict__ Q, + const char * __restrict__ K, + const char * __restrict__ V, + const char * __restrict__ mask, + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, + const float max_bias, + const float m0, + const float m1, + const uint32_t n_head_log2, + const int ne00, + const int ne01, + const int ne02, + const int ne03, + const int ne10, + const int ne11, + const int ne12, + const int ne13, + const int ne31, + const int nb31, + const int nb01, + const int nb02, + const int nb03, + const int nb11, + const int nb12, + const int nb13, + const int ne0, + const int ne1, + const int ne2, + const int ne3) { +#if FP16_AVAILABLE + //In this kernel Q, K, V are matrices while i, j, k are matrix indices. + + const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. + const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + + const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. + const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); + const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); + const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const half * maskh = (const half *) mask + ne11*ic0; + + const int stride_KV2 = nb11 / sizeof(half2); + + half slopeh = __float2half(1.0f); + + // ALiBi + if (max_bias > 0.0f) { + const uint32_t h = blockIdx.y; + + const float base = h < n_head_log2 ? m0 : m1; + const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slopeh = __float2half(powf(base, exph)); + } + + static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); + + __shared__ half KQ[ncols*FATTN_KQ_STRIDE_TILE_F16]; + half2 * KQ2 = (half2 *) KQ; + + __shared__ half2 KV_tmp[FATTN_KQ_STRIDE_TILE_F16][D/2 + 1]; // Pad D to avoid memory bank conflicts. + + half kqmax[ncols/nwarps]; +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + kqmax[j0/nwarps] = -HALF_MAX_HALF; + } + half2 kqsum[ncols/nwarps] = {{0.0f, 0.0f}}; + + half2 VKQ[ncols/nwarps][(D/2)/WARP_SIZE] = {{{0.0f, 0.0f}}}; + + // Convert Q to half2 and store in registers: + __shared__ half2 Q_h2[ncols][D/2]; +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + const float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i]; + Q_h2[j][i] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y); + } + } + + __syncthreads(); + + const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F16; + for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F16) { + // Calculate KQ tile and keep track of new maximum KQ values: + + half kqmax_new[ncols/nwarps]; +#pragma unroll + for (int j = 0; j < ncols/nwarps; ++j) { + kqmax_new[j] = kqmax[j]; + } + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += nwarps) { + const int i_KQ = i_KQ_0 + threadIdx.y; + +#pragma unroll + for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += WARP_SIZE) { + const int k_KQ = k_KQ_0 + threadIdx.x; + + KV_tmp[i_KQ][k_KQ] = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ]; + } + } + + __syncthreads(); + + half2 sum2[FATTN_KQ_STRIDE_TILE_F16/WARP_SIZE][ncols/nwarps] = {{{0.0f, 0.0f}}}; + +#pragma unroll + for (int k_KQ = 0; k_KQ < D/2; ++k_KQ) { + half2 K_k[FATTN_KQ_STRIDE_TILE_F16/WARP_SIZE]; + half2 Q_k[ncols/nwarps]; + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) { + const int i_KQ = i_KQ_0 + threadIdx.x; + + K_k[i_KQ_0/WARP_SIZE] = KV_tmp[i_KQ][k_KQ]; + } +#pragma unroll + for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) { + const int j_KQ = j_KQ_0 + threadIdx.y; + + Q_k[j_KQ_0/nwarps] = Q_h2[j_KQ][k_KQ]; + } + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) { +#pragma unroll + for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) { + sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += K_k[i_KQ_0/WARP_SIZE]*Q_k[j_KQ_0/nwarps]; + } + } + } + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F16; i_KQ_0 += WARP_SIZE) { + const int i_KQ = i_KQ_0 + threadIdx.x; + +#pragma unroll + for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) { + const int j_KQ = j_KQ_0 + threadIdx.y; + + half sum = __low2half(sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]) + __high2half(sum2[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]); + sum += mask ? slopeh*maskh[j_KQ*ne11 + k_VKQ_0 + i_KQ] : __float2half(0.0f); + + kqmax_new[j_KQ_0/nwarps] = ggml_cuda_hmax(kqmax_new[j_KQ_0/nwarps], sum); + + KQ[j_KQ*FATTN_KQ_STRIDE_TILE_F16 + i_KQ] = sum; + } + } + + __syncthreads(); + +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + kqmax_new[j0/nwarps] = warp_reduce_max(kqmax_new[j0/nwarps]); + const half2 KQ_max_scale = __half2half2(hexp(kqmax[j0/nwarps] - kqmax_new[j0/nwarps])); + kqmax[j0/nwarps] = kqmax_new[j0/nwarps]; + +#pragma unroll + for (int i0 = 0; i0 < FATTN_KQ_STRIDE_TILE_F16/2; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + const half2 diff = KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + i] - __half2half2(kqmax[j0/nwarps]); + const half2 val = h2exp(diff); + kqsum[j0/nwarps] = kqsum[j0/nwarps]*KQ_max_scale + val; + KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + i] = val; + } + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + VKQ[j0/nwarps][i0/WARP_SIZE] *= KQ_max_scale; + } + } + + __syncthreads(); + +#pragma unroll + for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F16; k0 += nwarps) { + const int k = k0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + KV_tmp[k][i] = V_h2[(k_VKQ_0 + k)*stride_KV2 + i]; + } + } + + __syncthreads(); + +#pragma unroll + for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F16; k0 += 2) { + half2 V_k[(D/2)/WARP_SIZE][2]; + half2 KQ_k[ncols/nwarps]; + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + V_k[i0/WARP_SIZE][0] = KV_tmp[k0 + 0][i]; + V_k[i0/WARP_SIZE][1] = KV_tmp[k0 + 1][i]; + } +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + KQ_k[j0/nwarps] = KQ2[j*(FATTN_KQ_STRIDE_TILE_F16/2) + k0/2]; + } + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + VKQ[j0/nwarps][i0/WARP_SIZE] += V_k[i0/WARP_SIZE][0]* __low2half2(KQ_k[j0/nwarps]); + VKQ[j0/nwarps][i0/WARP_SIZE] += V_k[i0/WARP_SIZE][1]*__high2half2(KQ_k[j0/nwarps]); + } + } + } + + __syncthreads(); + } + +#pragma unroll + for (int j_VKQ_0 = 0; j_VKQ_0 < ncols; j_VKQ_0 += nwarps) { + const int j_VKQ = j_VKQ_0 + threadIdx.y; + + half kqsum_j = __low2half(kqsum[j_VKQ_0/nwarps]) + __high2half(kqsum[j_VKQ_0/nwarps]); + kqsum_j = warp_reduce_sum(kqsum_j); + +#pragma unroll + for (int i00 = 0; i00 < D; i00 += 2*WARP_SIZE) { + const int i0 = i00 + 2*threadIdx.x; + + half2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)]; + if (parallel_blocks == 1) { + dst_val /= __half2half2(kqsum_j); + } + const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; + dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = __low2float(dst_val); + dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = __high2float(dst_val); + } + + if (parallel_blocks != 1 && threadIdx.x == 0) { + dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); + } + } +#else + NO_DEVICE_CODE; +#endif // FP16_AVAILABLE +} + +template void launch_fattn_tile_f16( + const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, + ggml_cuda_pool & pool, cudaStream_t main_stream +) { + ggml_cuda_pool_alloc dst_tmp(pool); + ggml_cuda_pool_alloc dst_tmp_meta(pool); + + if (parallel_blocks > 1) { + dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); + dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); + } + + constexpr int nwarps = 8; + const dim3 block_dim(WARP_SIZE, nwarps, 1); + const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); + const int shmem = 0; + + float scale = 1.0f; + float max_bias = 0.0f; + + memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); + memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); + + const uint32_t n_head = Q->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + flash_attn_tile_ext_f16 + <<>> ( + (const char *) Q->data, + (const char *) K->data, + (const char *) V->data, + mask ? ((const char *) mask->data) : nullptr, + parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, + scale, max_bias, m0, m1, n_head_log2, + Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], + K->ne[0], K->ne[1], K->ne[2], K->ne[3], + mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, + Q->nb[1], Q->nb[2], Q->nb[3], + K->nb[1], K->nb[2], K->nb[3], + KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] + ); + CUDA_CHECK(cudaGetLastError()); + + if (parallel_blocks == 1) { + return; + } + + const dim3 block_dim_combine(D, 1, 1); + const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); + const int shmem_combine = 0; + + flash_attn_combine_results + <<>> + (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); + CUDA_CHECK(cudaGetLastError()); +} + +void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + const ggml_tensor * mask = dst->src[3]; + + ggml_tensor * KQV = dst; + + const int32_t precision = KQV->op_params[2]; + GGML_ASSERT(precision == GGML_PREC_DEFAULT); + GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); + + if (Q->ne[1] <= 16) { + constexpr int cols_per_block = 16; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_tile_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_tile_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + if (Q->ne[1] <= 32) { + constexpr int cols_per_block = 32; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_tile_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_tile_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + constexpr int cols_per_block = 32; + constexpr int parallel_blocks = 1; + switch (Q->ne[0]) { + case 64: + launch_fattn_tile_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_tile_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } +} diff --git a/ggml-cuda/fattn-tile-f16.cuh b/ggml-cuda/fattn-tile-f16.cuh new file mode 100644 index 000000000..ffc587842 --- /dev/null +++ b/ggml-cuda/fattn-tile-f16.cuh @@ -0,0 +1,3 @@ +#include "common.cuh" + +void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml-cuda/fattn-tile-f32.cu b/ggml-cuda/fattn-tile-f32.cu new file mode 100644 index 000000000..176895edd --- /dev/null +++ b/ggml-cuda/fattn-tile-f32.cu @@ -0,0 +1,393 @@ +#include "common.cuh" +#include "fattn-common.cuh" +#include "fattn-tile-f32.cuh" + +#define FATTN_KQ_STRIDE_TILE_F32 32 + +template // D == head size +#if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +__launch_bounds__(nwarps*WARP_SIZE, 1) +#endif // !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) +static __global__ void flash_attn_tile_ext_f32( + const char * __restrict__ Q, + const char * __restrict__ K, + const char * __restrict__ V, + const char * __restrict__ mask, + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, + const float max_bias, + const float m0, + const float m1, + const uint32_t n_head_log2, + const int ne00, + const int ne01, + const int ne02, + const int ne03, + const int ne10, + const int ne11, + const int ne12, + const int ne13, + const int ne31, + const int nb31, + const int nb01, + const int nb02, + const int nb03, + const int nb11, + const int nb12, + const int nb13, + const int ne0, + const int ne1, + const int ne2, + const int ne3) { + //In this kernel Q, K, V are matrices while i, j, k are matrix indices. + + const int ic0 = (blockIdx.x / parallel_blocks) * ncols; // Index of the Q/QKV column to work on. + const int ip = blockIdx.x % parallel_blocks; // Index in group of blocks running for the same column in parallel. + + const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix. + const float2 * Q_f2 = (const float2 *) (Q + nb02* blockIdx.y + nb01*ic0); + const half2 * K_h2 = (const half2 *) (K + nb12*(blockIdx.y / gqa_ratio)); + const half2 * V_h2 = (const half2 *) (V + nb12*(blockIdx.y / gqa_ratio)); // K and V have same shape + const half * maskh = (const half *) mask + ne11*ic0; + + const int stride_KV2 = nb11 / sizeof(half2); + + float slope = 1.0f; + + // ALiBi + if (max_bias > 0.0f) { + const uint32_t h = blockIdx.y; + + const float base = h < n_head_log2 ? m0 : m1; + const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + slope = powf(base, exph); + } + + static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); + + __shared__ float KQ[ncols*FATTN_KQ_STRIDE_TILE_F32]; + + __shared__ float KV_tmp[FATTN_KQ_STRIDE_TILE_F32][D + 1]; // Pad D to avoid memory bank conflicts. + float2 * KV_tmp2 = (float2 *) KV_tmp; + + float kqmax[ncols/nwarps]; +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + kqmax[j0/nwarps] = -FLT_MAX/2.0f; + } + float kqsum[ncols/nwarps] = {0.0f}; + + float2 VKQ[ncols/nwarps][(D/2)/WARP_SIZE] = {{{0.0f, 0.0f}}}; + + // Convert Q to half2 and store in registers: + __shared__ float Q_f[ncols][D]; +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + const int j = j0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < D; i0 += 2*WARP_SIZE) { + float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i0/2 + threadIdx.x]; + Q_f[j][i0 + 0*WARP_SIZE + threadIdx.x] = tmp.x * scale; + Q_f[j][i0 + 1*WARP_SIZE + threadIdx.x] = tmp.y * scale; + } + } + + __syncthreads(); + + const int k_start = parallel_blocks == 1 ? 0 : ip*FATTN_KQ_STRIDE_TILE_F32; + for (int k_VKQ_0 = k_start; k_VKQ_0 < ne11; k_VKQ_0 += parallel_blocks*FATTN_KQ_STRIDE_TILE_F32) { + // Calculate KQ tile and keep track of new maximum KQ values: + + float kqmax_new[ncols/nwarps]; +#pragma unroll + for (int j = 0; j < ncols/nwarps; ++j) { + kqmax_new[j] = kqmax[j]; + } + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += nwarps) { + const int i_KQ = i_KQ_0 + threadIdx.y; + +#pragma unroll + for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 2*WARP_SIZE) { + const half2 tmp = K_h2[(k_VKQ_0 + i_KQ)*stride_KV2 + k_KQ_0/2 + threadIdx.x]; + KV_tmp[i_KQ][k_KQ_0 + 0*WARP_SIZE + threadIdx.x] = __low2float(tmp); + KV_tmp[i_KQ][k_KQ_0 + 1*WARP_SIZE + threadIdx.x] = __high2float(tmp); + } + } + + __syncthreads(); + + float sum[FATTN_KQ_STRIDE_TILE_F32/WARP_SIZE][ncols/nwarps] = {{0.0f}}; + +#pragma unroll + for (int k_KQ = 0; k_KQ < D; ++k_KQ) { + float K_k[FATTN_KQ_STRIDE_TILE_F32/WARP_SIZE]; + float Q_k[ncols/nwarps]; + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) { + const int i_KQ = i_KQ_0 + threadIdx.x; + + K_k[i_KQ_0/WARP_SIZE] = KV_tmp[i_KQ][k_KQ]; + } +#pragma unroll + for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) { + const int j_KQ = j_KQ_0 + threadIdx.y; + + Q_k[j_KQ_0/nwarps] = Q_f[j_KQ][k_KQ]; + } + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) { +#pragma unroll + for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) { + sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += K_k[i_KQ_0/WARP_SIZE] * Q_k[j_KQ_0/nwarps]; + } + } + } + +#pragma unroll + for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE_TILE_F32; i_KQ_0 += WARP_SIZE) { + const int i_KQ = i_KQ_0 + threadIdx.x; + +#pragma unroll + for (int j_KQ_0 = 0; j_KQ_0 < ncols; j_KQ_0 += nwarps) { + const int j_KQ = j_KQ_0 + threadIdx.y; + + sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps] += mask ? slope*__half2float(maskh[j_KQ*ne11 + k_VKQ_0 + i_KQ]) : 0.0f; + + kqmax_new[j_KQ_0/nwarps] = fmaxf(kqmax_new[j_KQ_0/nwarps], sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]); + + KQ[j_KQ*FATTN_KQ_STRIDE_TILE_F32 + i_KQ] = sum[i_KQ_0/WARP_SIZE][j_KQ_0/nwarps]; + } + } + + __syncthreads(); + +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + kqmax_new[j0/nwarps] = warp_reduce_max(kqmax_new[j0/nwarps]); + const float KQ_max_scale = expf(kqmax[j0/nwarps] - kqmax_new[j0/nwarps]); + kqmax[j0/nwarps] = kqmax_new[j0/nwarps]; + + float kqsum_add = 0.0f; +#pragma unroll + for (int i0 = 0; i0 < FATTN_KQ_STRIDE_TILE_F32; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + const float diff = KQ[j*FATTN_KQ_STRIDE_TILE_F32 + i] - kqmax[j0/nwarps]; + const float val = expf(diff); + kqsum_add += val; + KQ[j*FATTN_KQ_STRIDE_TILE_F32 + i] = val; + } + kqsum[j0/nwarps] = kqsum[j0/nwarps]*KQ_max_scale + kqsum_add; + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + VKQ[j0/nwarps][i0/WARP_SIZE].x *= KQ_max_scale; + VKQ[j0/nwarps][i0/WARP_SIZE].y *= KQ_max_scale; + } + } + + __syncthreads(); + +#pragma unroll + for (int k0 = 0; k0 < FATTN_KQ_STRIDE_TILE_F32; k0 += nwarps) { + const int k = k0 + threadIdx.y; + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + KV_tmp2[k*(D/2) + i].x = __low2float(V_h2[(k_VKQ_0 + k)*stride_KV2 + i]); + KV_tmp2[k*(D/2) + i].y = __high2float(V_h2[(k_VKQ_0 + k)*stride_KV2 + i]); + } + } + + __syncthreads(); + +#pragma unroll + for (int k = 0; k < FATTN_KQ_STRIDE_TILE_F32; ++k) { + float2 V_k[(D/2)/WARP_SIZE]; + float KQ_k[ncols/nwarps]; + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { + const int i = i0 + threadIdx.x; + + V_k[i0/WARP_SIZE] = KV_tmp2[k*(D/2) + i]; + } +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + const int j = j0 + threadIdx.y; + + KQ_k[j0/nwarps] = KQ[j*FATTN_KQ_STRIDE_TILE_F32 + k]; + } + +#pragma unroll + for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { +#pragma unroll + for (int j0 = 0; j0 < ncols; j0 += nwarps) { + VKQ[j0/nwarps][i0/WARP_SIZE].x += V_k[i0/WARP_SIZE].x*KQ_k[j0/nwarps]; + VKQ[j0/nwarps][i0/WARP_SIZE].y += V_k[i0/WARP_SIZE].y*KQ_k[j0/nwarps]; + } + } + } + + __syncthreads(); + } + +#pragma unroll + for (int j_VKQ_0 = 0; j_VKQ_0 < ncols; j_VKQ_0 += nwarps) { + const int j_VKQ = j_VKQ_0 + threadIdx.y; + + float kqsum_j = kqsum[j_VKQ_0/nwarps]; + kqsum_j = warp_reduce_sum(kqsum_j); + +#pragma unroll + for (int i00 = 0; i00 < D; i00 += 2*WARP_SIZE) { + const int i0 = i00 + 2*threadIdx.x; + + float2 dst_val = VKQ[j_VKQ_0/nwarps][i0/(2*WARP_SIZE)]; + if (parallel_blocks == 1) { + dst_val.x /= kqsum_j; + dst_val.y /= kqsum_j; + } + const int j_dst = (ic0 + j_VKQ)*parallel_blocks + ip; + dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 0] = dst_val.x; + dst[j_dst*D*gridDim.y + D*blockIdx.y + i0 + 1] = dst_val.y; + } + + if (parallel_blocks != 1 && threadIdx.x == 0) { + dst_meta[(ic0 + j_VKQ)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j_VKQ_0/nwarps], kqsum_j); + } + } +} + +template void launch_fattn_tile_f32( + const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, + ggml_cuda_pool & pool, cudaStream_t main_stream +) { + ggml_cuda_pool_alloc dst_tmp(pool); + ggml_cuda_pool_alloc dst_tmp_meta(pool); + + if (parallel_blocks > 1) { + dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); + dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); + } + + constexpr int nwarps = 8; + const dim3 block_dim(WARP_SIZE, nwarps, 1); + const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); + const int shmem = 0; + + float scale = 1.0f; + float max_bias = 0.0f; + + memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); + memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); + + const uint32_t n_head = Q->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + flash_attn_tile_ext_f32 + <<>> ( + (const char *) Q->data, + (const char *) K->data, + (const char *) V->data, + mask ? ((const char *) mask->data) : nullptr, + parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, + scale, max_bias, m0, m1, n_head_log2, + Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], + K->ne[0], K->ne[1], K->ne[2], K->ne[3], + mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, + Q->nb[1], Q->nb[2], Q->nb[3], + K->nb[1], K->nb[2], K->nb[3], + KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] + ); + CUDA_CHECK(cudaGetLastError()); + + if (parallel_blocks == 1) { + return; + } + + const dim3 block_dim_combine(D, 1, 1); + const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); + const int shmem_combine = 0; + + flash_attn_combine_results + <<>> + (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); + CUDA_CHECK(cudaGetLastError()); +} + +void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + const ggml_tensor * mask = dst->src[3]; + + ggml_tensor * KQV = dst; + + const int32_t precision = KQV->op_params[2]; + GGML_ASSERT(precision == GGML_PREC_DEFAULT); + GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); + + if (Q->ne[1] <= 16) { + constexpr int cols_per_block = 16; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_tile_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_tile_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + if (Q->ne[1] <= 32) { + constexpr int cols_per_block = 32; + constexpr int parallel_blocks = 4; + switch (Q->ne[0]) { + case 64: + launch_fattn_tile_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_tile_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } + return; + } + + constexpr int cols_per_block = 32; + constexpr int parallel_blocks = 1; + switch (Q->ne[0]) { + case 64: + launch_fattn_tile_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + case 128: + launch_fattn_tile_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); + break; + default: + GGML_ASSERT(false); + break; + } +} diff --git a/ggml-cuda/fattn-tile-f32.cuh b/ggml-cuda/fattn-tile-f32.cuh new file mode 100644 index 000000000..b1c546c80 --- /dev/null +++ b/ggml-cuda/fattn-tile-f32.cuh @@ -0,0 +1,3 @@ +#include "common.cuh" + +void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/ggml-cuda/fattn-vec-f16.cu b/ggml-cuda/fattn-vec-f16.cu index cbf5f7835..a18be5ddc 100644 --- a/ggml-cuda/fattn-vec-f16.cu +++ b/ggml-cuda/fattn-vec-f16.cu @@ -57,7 +57,7 @@ static __global__ void flash_attn_vec_ext_f16( // ALiBi if (max_bias > 0.0f) { - const int h = blockIdx.y; + const uint32_t h = blockIdx.y; const float base = h < n_head_log2 ? m0 : m1; const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; @@ -232,11 +232,8 @@ static __global__ void flash_attn_vec_ext_f16( dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; } - if (parallel_blocks != 1 && tid != 0) { -#pragma unroll - for (int j = 0; j < ncols; ++j) { - dst_meta[(ic0 + j)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j], kqsum[j]); - } + if (parallel_blocks != 1 && threadIdx.x < ncols) { + dst_meta[(ic0 + threadIdx.x)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[threadIdx.x], kqsum[threadIdx.x]); } #else NO_DEVICE_CODE; diff --git a/ggml-cuda/fattn-vec-f32.cu b/ggml-cuda/fattn-vec-f32.cu index 40c336ce3..91fcdc8c3 100644 --- a/ggml-cuda/fattn-vec-f32.cu +++ b/ggml-cuda/fattn-vec-f32.cu @@ -56,7 +56,7 @@ static __global__ void flash_attn_vec_ext_f32( // ALiBi if (max_bias > 0.0f) { - const int h = blockIdx.y; + const uint32_t h = blockIdx.y; const float base = h < n_head_log2 ? m0 : m1; const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; @@ -221,11 +221,8 @@ static __global__ void flash_attn_vec_ext_f32( dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; } - if (parallel_blocks != 1 && tid != 0) { -#pragma unroll - for (int j = 0; j < ncols; ++j) { - dst_meta[(ic0 + j)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[j], kqsum[j]); - } + if (parallel_blocks != 1 && threadIdx.x < ncols) { + dst_meta[(ic0 + threadIdx.x)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[threadIdx.x], kqsum[threadIdx.x]); } } diff --git a/ggml-cuda/fattn.cu b/ggml-cuda/fattn.cu index 419f8e752..a1918e258 100644 --- a/ggml-cuda/fattn.cu +++ b/ggml-cuda/fattn.cu @@ -1,5 +1,7 @@ #include "common.cuh" #include "fattn-common.cuh" +#include "fattn-tile-f16.cuh" +#include "fattn-tile-f32.cuh" #include "fattn-vec-f16.cuh" #include "fattn-vec-f32.cuh" #include "fattn.cuh" @@ -88,7 +90,7 @@ static __global__ void flash_attn_ext_f16( // ALiBi if (max_bias > 0.0f) { - const int h = blockIdx.y; + const uint32_t h = blockIdx.y; const float base = h < n_head_log2 ? m0 : m1; const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; @@ -541,13 +543,31 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst const int32_t precision = KQV->op_params[2]; + // On AMD the tile kernels perform poorly, use the vec kernel instead: + if (cc >= CC_OFFSET_AMD) { + if (precision == GGML_PREC_DEFAULT) { + ggml_cuda_flash_attn_ext_vec_f16_no_mma(ctx, dst); + } else { + ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); + } + return; + } + if (!fast_fp16_available(cc)) { - ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); + if (Q->ne[1] <= 8) { + ggml_cuda_flash_attn_ext_vec_f32(ctx, dst); + } else { + ggml_cuda_flash_attn_ext_tile_f32(ctx, dst); + } return; } if (!fp16_mma_available(cc)) { - ggml_cuda_flash_attn_ext_vec_f16_no_mma(ctx, dst); + if (Q->ne[1] <= 8) { + ggml_cuda_flash_attn_ext_vec_f16_no_mma(ctx, dst); + } else { + ggml_cuda_flash_attn_ext_tile_f16(ctx, dst); + } return; } From b43272afa29a64dcb8bcf26a96a05bac40792b92 Mon Sep 17 00:00:00 2001 From: jaime-m-p <167997752+jaime-m-p@users.noreply.github.com> Date: Sat, 18 May 2024 01:09:13 +0200 Subject: [PATCH 055/137] Unicode codepoint flags for custom regexs (#7245) * Replace CODEPOINT_TYPE_* with codepoint_flags * Update and bugfix brute force random test * Deterministic brute force random test * Unicode normalization NFD * Get rid of BOM --- llama.cpp | 8 +- scripts/gen-unicode-data.py | 166 +- tests/test-tokenizer-random.py | 113 +- unicode-data.cpp | 9138 ++++++++++++++++++++++++-------- unicode-data.h | 27 +- unicode.cpp | 200 +- unicode.h | 56 +- 7 files changed, 7299 insertions(+), 2409 deletions(-) diff --git a/llama.cpp b/llama.cpp index e11f0ac4b..b752ddc6b 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12576,16 +12576,16 @@ struct llm_tokenizer_wpm { // to lowercase, pad chinese characters, pad punctuation std::string new_str = ""; for (uint32_t code : cpts_nfd) { - int type = unicode_cpt_type(code); - if (type == CODEPOINT_TYPE_ACCENT_MARK || type == CODEPOINT_TYPE_CONTROL) { + const codepoint_flags flags = unicode_cpt_flags(code); + if (flags.is_accent_mark || flags.is_control) { continue; } code = unicode_tolower(code); - if (type == CODEPOINT_TYPE_SEPARATOR) { + if (flags.is_separator || flags.is_whitespace) { //####FIXME: is_separator ? code = ' '; } std::string s = unicode_cpt_to_utf8(code); - if (type == CODEPOINT_TYPE_PUNCTUATION || is_ascii_punct(code) || is_chinese_char(code)) { + if (flags.is_punctuation || is_ascii_punct(code) || is_chinese_char(code)) { new_str += " "; new_str += s; new_str += " "; diff --git a/scripts/gen-unicode-data.py b/scripts/gen-unicode-data.py index 37d1e396c..744873c2a 100644 --- a/scripts/gen-unicode-data.py +++ b/scripts/gen-unicode-data.py @@ -1,64 +1,134 @@ import regex +import ctypes +import unicodedata -def get_matches(regex_expr): - regex_expr_compiled = regex.compile(regex_expr) - unicode_ranges = [] - current_range = None - - for codepoint in range(0x110000): - char = chr(codepoint) - if regex_expr_compiled.match(char): - if current_range is None: - current_range = [codepoint, codepoint] - else: - current_range[1] = codepoint - elif current_range is not None: - unicode_ranges.append(tuple(current_range)) - current_range = None - - if current_range is not None: - unicode_ranges.append(tuple(current_range)) - - return unicode_ranges +class CoodepointFlags (ctypes.Structure): + _fields_ = [ # see definition in unicode.h + ("is_undefined", ctypes.c_uint16, 1), + ("is_number", ctypes.c_uint16, 1), # regex: \p{N} + ("is_letter", ctypes.c_uint16, 1), # regex: \p{L} + ("is_separator", ctypes.c_uint16, 1), # regex: \p{Z} + ("is_accent_mark", ctypes.c_uint16, 1), # regex: \p{M} + ("is_punctuation", ctypes.c_uint16, 1), # regex: \p{P} + ("is_symbol", ctypes.c_uint16, 1), # regex: \p{S} + ("is_control", ctypes.c_uint16, 1), # regex: \p{C} + ] -def print_cat(mode, cat, ranges): - if mode == "range": - print("const std::vector> unicode_ranges_{} = {{".format(cat)) # noqa: NP100 - if mode == "map": - print("const std::map unicode_map_{} = {{".format(cat)) # noqa: NP100 - for i, values in enumerate(ranges): - end = ",\n" if (i % 4 == 3 or i + 1 == len(ranges)) else ", " - values = ["0x%08X" % value for value in values] - print("{" + ", ".join(values) + "}", end=end) # noqa: NP100 - print("};") # noqa: NP100 - print("") # noqa: NP100 +assert (ctypes.sizeof(CoodepointFlags) == 2) -print_cat("range", "number", get_matches(r'\p{N}')) -print_cat("range", "letter", get_matches(r'\p{L}')) -print_cat("range", "separator", get_matches(r'\p{Z}')) -print_cat("range", "accent_mark", get_matches(r'\p{M}')) -print_cat("range", "punctuation", get_matches(r'\p{P}')) -print_cat("range", "symbol", get_matches(r'\p{S}')) -print_cat("range", "control", get_matches(r'\p{C}')) +MAX_CODEPOINTS = 0x110000 -print_cat("range", "whitespace", get_matches(r'\s')) +regex_number = regex.compile(r'\p{N}') +regex_letter = regex.compile(r'\p{L}') +regex_separator = regex.compile(r'\p{Z}') +regex_accent_mark = regex.compile(r'\p{M}') +regex_punctuation = regex.compile(r'\p{P}') +regex_symbol = regex.compile(r'\p{S}') +regex_control = regex.compile(r'\p{C}') +regex_whitespace = regex.compile(r'\s') +codepoint_flags = (CoodepointFlags * MAX_CODEPOINTS)() +table_whitespace = [] +table_lowercase = [] +table_uppercase = [] +table_nfd = [] -map_lowercase = [] -map_uppercase = [] -for codepoint in range(0x110000): +for codepoint in range(MAX_CODEPOINTS): + # convert codepoint to unicode character char = chr(codepoint) + + # regex categories + flags = codepoint_flags[codepoint] + flags.is_number = bool(regex_number.match(char)) + flags.is_letter = bool(regex_letter.match(char)) + flags.is_separator = bool(regex_separator.match(char)) + flags.is_accent_mark = bool(regex_accent_mark.match(char)) + flags.is_punctuation = bool(regex_punctuation.match(char)) + flags.is_symbol = bool(regex_symbol.match(char)) + flags.is_control = bool(regex_control.match(char)) + flags.is_undefined = bytes(flags)[0] == 0 + assert (not flags.is_undefined) + + # whitespaces + if bool(regex_whitespace.match(char)): + table_whitespace.append(codepoint) + + # lowercase conversion lower = ord(char.lower()[0]) - upper = ord(char.upper()[0]) if codepoint != lower: - map_lowercase.append((codepoint, lower)) + table_lowercase.append((codepoint, lower)) + + # uppercase conversion + upper = ord(char.upper()[0]) if codepoint != upper: - map_uppercase.append((codepoint, upper)) -print_cat("map", "lowercase", map_lowercase) -print_cat("map", "uppercase", map_uppercase) + table_uppercase.append((codepoint, upper)) + + # NFD normalization + norm = ord(unicodedata.normalize('NFD', char)[0]) + if codepoint != norm: + table_nfd.append((codepoint, norm)) -# TODO: generate unicode_map_nfd +# group ranges with same flags +ranges_flags = [(0, codepoint_flags[0])] # start, flags +for codepoint, flags in enumerate(codepoint_flags): + if bytes(flags) != bytes(ranges_flags[-1][1]): + ranges_flags.append((codepoint, flags)) +ranges_flags.append((MAX_CODEPOINTS, CoodepointFlags())) + + +# group ranges with same nfd +ranges_nfd = [(0, 0, 0)] # start, last, nfd +for codepoint, norm in table_nfd: + start = ranges_nfd[-1][0] + if ranges_nfd[-1] != (start, codepoint - 1, norm): + ranges_nfd.append(None) + start = codepoint + ranges_nfd[-1] = (start, codepoint, norm) + + +# Generate 'unicode-data.cpp' + + +def out(line=""): + print(line, end='\n') # noqa + + +out("""\ +// generated with scripts/gen-unicode-data.py + +#include "unicode-data.h" + +#include +#include +#include +#include +""") + +out("const std::vector> unicode_ranges_flags = { // start, flags // last=next_start-1") +for codepoint, flags in ranges_flags: + flags = int.from_bytes(bytes(flags), "little") + out("{0x%06X, 0x%04X}," % (codepoint, flags)) +out("};\n") + +out("const std::unordered_set unicode_set_whitespace = {") +out(", ".join("0x%06X" % cpt for cpt in table_whitespace)) +out("};\n") + +out("const std::unordered_map unicode_map_lowercase = {") +for tuple in table_lowercase: + out("{0x%06X, 0x%06X}," % tuple) +out("};\n") + +out("const std::unordered_map unicode_map_uppercase = {") +for tuple in table_uppercase: + out("{0x%06X, 0x%06X}," % tuple) +out("};\n") + +out("const std::vector unicode_ranges_nfd = { // start, last, nfd") +for triple in ranges_nfd: + out("{0x%06X, 0x%06X, 0x%06X}," % triple) +out("};\n") diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py index 5b2ab8ef7..d5a6f185f 100644 --- a/tests/test-tokenizer-random.py +++ b/tests/test-tokenizer-random.py @@ -1,5 +1,5 @@ # Test libllama tokenizer == AutoTokenizer. -# Brute force random tokens/text generation. +# Brute force random words/text generation. # # Sample usage: # @@ -12,10 +12,10 @@ import argparse import subprocess import random -from typing import Iterator +from typing import Callable, Iterator import cffi -from transformers import AutoTokenizer, PreTrainedTokenizerBase +from transformers import AutoTokenizer logger = logging.getLogger("test-tokenizer-random-bpe") @@ -145,28 +145,35 @@ def generator_custom_text() -> Iterator[str]: def generator_custom_text_edge_cases() -> Iterator[str]: """Edge cases found while debugging""" yield from [ - '\x1f-a', # unicode_ranges_control, {0x00001C, 0x00001F} - '¼-a', # unicode_ranges_digit, 0x00BC - '½-a', # unicode_ranges_digit, 0x00BD - '¾-a', # unicode_ranges_digit, 0x00BE - 'a 〇b', # unicode_ranges_digit, 0x3007 - 'Ⅵ-a', # unicode_ranges_digit, {0x00002150, 0x0000218F} // Number Forms - '\uFEFF//', # unicode_ranges_control, 0xFEFF (BOM) - 'a' # TODO: Phi-3 fail + '\x1f-a', # unicode_ranges_control, {0x00001C, 0x00001F} + '¼-a', # unicode_ranges_digit, 0x00BC + '½-a', # unicode_ranges_digit, 0x00BD + '¾-a', # unicode_ranges_digit, 0x00BE + 'a 〇b', # unicode_ranges_digit, 0x3007 + 'Ⅵ-a', # unicode_ranges_digit, {0x00002150, 0x0000218F} // Number Forms + '\uFEFF//', # unicode_ranges_control, 0xFEFF (BOM) + 'Cửa Việt', # llama-3, ignore_merges = true + 'a', # TODO: Phi-3 fail + 'a\na', # TODO: Bert fail ] -def generator_random_chars(iterations = 100) -> Iterator[str]: +def generator_vocab_words(vocab: list[str]) -> Iterator[str]: + """Brute force check all vocab words""" + yield from vocab + + +def generator_random_chars(iterations=100) -> Iterator[str]: """Brute force random text with simple characters""" WHITESPACES = list(" " * 20 + "\n" * 5 + "\r\n" * 5 + "\t" * 5) - CHARS = list(set(""" + CHARS = list(sorted(set(""" ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz ÁÉÍÓÚÀÈÌÒÙÂÊÎÔÛÄËÏÖÜ áéíóúàèìòùâêîôûäëïöü .-,*/-+ª!"·$%&/()=?¿[]{}<>\\|@#~½¬~;:_ - """)) + """))) rand = random.Random() for m in range(iterations): @@ -181,13 +188,13 @@ def generator_random_chars(iterations = 100) -> Iterator[str]: yield "".join(text) -def generator_random_vocab_chars(tokenizer: PreTrainedTokenizerBase, iterations = 100) -> Iterator[str]: +def generator_random_vocab_chars(vocab: list[str], iterations=100) -> Iterator[str]: """Brute force random text with vocab characters""" - vocab_ids = list(tokenizer.vocab.values()) - vocab_text = tokenizer.decode(vocab_ids, skip_special_tokens=True) - vocab_chars = list(set(vocab_text)) - del vocab_ids, vocab_text + vocab_chars = set() + for word in vocab: + vocab_chars.update(word) + vocab_chars = list(sorted(vocab_chars)) rand = random.Random() for m in range(iterations): @@ -196,19 +203,11 @@ def generator_random_vocab_chars(tokenizer: PreTrainedTokenizerBase, iterations yield "".join(text) -def generator_random_vocab_tokens(tokenizer: PreTrainedTokenizerBase, iterations = 100) -> Iterator[str]: - """Brute force random text from vocab tokens""" +def generator_random_vocab_words(vocab: list[str], iterations=100) -> Iterator[str]: + """Brute force random text from vocab words""" - space_id = tokenizer.encode(" ", add_special_tokens=False)[0] - vocab_ids = list(tokenizer.vocab.values()) - vocab_ids = list(sorted(vocab_ids + vocab_ids)) - for i in range(1, len(vocab_ids), 2): - vocab_ids[i] = space_id - vocab_tokens = tokenizer.decode(vocab_ids, skip_special_tokens=True) - vocab_tokens = vocab_tokens.split(" ") - del vocab_ids - - yield from vocab_tokens + vocab = [w.strip() for w in vocab] + yield from vocab rand = random.Random() for m in range(iterations): @@ -217,14 +216,13 @@ def generator_random_vocab_tokens(tokenizer: PreTrainedTokenizerBase, iterations num_words = rand.randint(300, 400) for i in range(num_words): k = rand.randint(1, 3) - tokens = rand.choices(vocab_tokens, k=k) - tokens = [t.strip(" \n\r\t") for t in tokens] + words = rand.choices(vocab, k=k) sep = rand.choice(" \n\r\t") - text.append("".join(tokens) + sep) + text.append("".join(words) + sep) yield "".join(text) -def generator_random_bytes(iterations = 100) -> Iterator[str]: +def generator_random_bytes(iterations=100) -> Iterator[str]: """Brute force random bytes""" WHITESPACES = list(" " * 20 + "\n" * 5 + "\r\n" * 5 + "\t" * 5) @@ -242,10 +240,10 @@ def generator_random_bytes(iterations = 100) -> Iterator[str]: yield "".join(text) -def test_compare_tokenizer(model: LibLlamaModel, tokenizer: PreTrainedTokenizerBase, generator: Iterator[str]): +def test_compare_tokenizer(func_tokenize1: Callable, func_tokenize2: Callable, generator: Iterator[str]): def find_first_mismatch(ids1: list[int], ids2: list[int]): - for i, (a,b) in enumerate(zip(ids1, ids2)): + for i, (a, b) in enumerate(zip(ids1, ids2)): if a != b: return i if len(ids1) == len(ids2): @@ -255,15 +253,12 @@ def test_compare_tokenizer(model: LibLlamaModel, tokenizer: PreTrainedTokenizerB t0 = time.perf_counter() logger.info("%s: %s" % (generator.__name__, "ini")) for text in generator: - ids1 = model.tokenize(text, add_special=False, parse_special=False) - ids2 = tokenizer.encode(text, add_special_tokens=False) + ids1 = func_tokenize1(text) + ids2 = func_tokenize2(text) if ids1 != ids2: i = find_first_mismatch(ids1, ids2) ids1 = list(ids1)[max(0, i - 2) : i + 2 + 1] ids2 = list(ids2)[max(0, i - 2) : i + 2 + 1] - text2 = tokenizer.decode(ids2, skip_special_tokens=True) - assert (text2 in text) - logger.info(" Text: " + repr(text2)) logger.info(" TokenIDs: " + str(ids1)) logger.info(" Expected: " + str(ids2)) raise Exception() @@ -271,25 +266,37 @@ def test_compare_tokenizer(model: LibLlamaModel, tokenizer: PreTrainedTokenizerB logger.info("%s: end, time: %.3f secs" % (generator.__name__, t1 - t0)) -if __name__ == "__main__": - +def main(argv: list[str] = None): parser = argparse.ArgumentParser() parser.add_argument("vocab_file", help="path to vocab 'gguf' file") parser.add_argument("dir_tokenizer", help="directory containing 'tokenizer.model' file") parser.add_argument("--verbose", action="store_true", help="increase output verbosity") - args = parser.parse_args() + args = parser.parse_args(argv) logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) - model = LibLlamaModel(LibLlama(), args.vocab_file, mparams=dict(vocab_only=True), cparams=dict(n_ctx=2048)) - + model = LibLlamaModel(LibLlama(), args.vocab_file, mparams=dict(vocab_only=True), cparams=dict(n_ctx=4096)) tokenizer = AutoTokenizer.from_pretrained(args.dir_tokenizer) - test_compare_tokenizer(model, tokenizer, generator_custom_text()) - test_compare_tokenizer(model, tokenizer, generator_custom_text_edge_cases()) - test_compare_tokenizer(model, tokenizer, generator_random_chars(10_000)) - test_compare_tokenizer(model, tokenizer, generator_random_vocab_chars(tokenizer, 10_000)) - test_compare_tokenizer(model, tokenizer, generator_random_vocab_tokens(tokenizer, 10_000)) - # test_compare_tokenizer(model, tokenizer, generator_random_bytes(10_000)) # FAIL + def func_tokenize2(text: str): + return tokenizer.encode(text, add_special_tokens=False) + + parse_special = all(len(func_tokenize2(t)) == 1 for t in tokenizer.all_special_tokens) + + def func_tokenize1(text: str): + return model.tokenize(text, add_special=False, parse_special=parse_special) + + vocab = list(sorted(tokenizer.batch_decode(list(tokenizer.get_vocab().values()), skip_special_tokens=True))) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_custom_text()) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_custom_text_edge_cases()) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_vocab_words(vocab)) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_chars(10_000)) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_vocab_chars(vocab, 10_000)) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_vocab_words(vocab, 10_000)) + # test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_bytes(10_000)) # FAIL model.free() + + +if __name__ == "__main__": + main() diff --git a/unicode-data.cpp b/unicode-data.cpp index c54175fc3..d7c1c898d 100644 --- a/unicode-data.cpp +++ b/unicode-data.cpp @@ -1,2183 +1,6983 @@ -#include "unicode-data.h" +// generated with scripts/gen-unicode-data.py + +#include "unicode-data.h" #include -#include -#include #include +#include +#include -// generated with scripts/gen-unicode-data.py -// -// TODO: generate unicode_map_nfd - -const std::vector> unicode_ranges_number = { -{0x00000030, 0x00000039}, {0x000000B2, 0x000000B3}, {0x000000B9, 0x000000B9}, {0x000000BC, 0x000000BE}, -{0x00000660, 0x00000669}, {0x000006F0, 0x000006F9}, {0x000007C0, 0x000007C9}, {0x00000966, 0x0000096F}, -{0x000009E6, 0x000009EF}, {0x000009F4, 0x000009F9}, {0x00000A66, 0x00000A6F}, {0x00000AE6, 0x00000AEF}, -{0x00000B66, 0x00000B6F}, {0x00000B72, 0x00000B77}, {0x00000BE6, 0x00000BF2}, {0x00000C66, 0x00000C6F}, -{0x00000C78, 0x00000C7E}, {0x00000CE6, 0x00000CEF}, {0x00000D58, 0x00000D5E}, {0x00000D66, 0x00000D78}, -{0x00000DE6, 0x00000DEF}, {0x00000E50, 0x00000E59}, {0x00000ED0, 0x00000ED9}, {0x00000F20, 0x00000F33}, -{0x00001040, 0x00001049}, {0x00001090, 0x00001099}, {0x00001369, 0x0000137C}, {0x000016EE, 0x000016F0}, -{0x000017E0, 0x000017E9}, {0x000017F0, 0x000017F9}, {0x00001810, 0x00001819}, {0x00001946, 0x0000194F}, -{0x000019D0, 0x000019DA}, {0x00001A80, 0x00001A89}, {0x00001A90, 0x00001A99}, {0x00001B50, 0x00001B59}, -{0x00001BB0, 0x00001BB9}, {0x00001C40, 0x00001C49}, {0x00001C50, 0x00001C59}, {0x00002070, 0x00002070}, -{0x00002074, 0x00002079}, {0x00002080, 0x00002089}, {0x00002150, 0x00002182}, {0x00002185, 0x00002189}, -{0x00002460, 0x0000249B}, {0x000024EA, 0x000024FF}, {0x00002776, 0x00002793}, {0x00002CFD, 0x00002CFD}, -{0x00003007, 0x00003007}, {0x00003021, 0x00003029}, {0x00003038, 0x0000303A}, {0x00003192, 0x00003195}, -{0x00003220, 0x00003229}, {0x00003248, 0x0000324F}, {0x00003251, 0x0000325F}, {0x00003280, 0x00003289}, -{0x000032B1, 0x000032BF}, {0x0000A620, 0x0000A629}, {0x0000A6E6, 0x0000A6EF}, {0x0000A830, 0x0000A835}, -{0x0000A8D0, 0x0000A8D9}, {0x0000A900, 0x0000A909}, {0x0000A9D0, 0x0000A9D9}, {0x0000A9F0, 0x0000A9F9}, -{0x0000AA50, 0x0000AA59}, {0x0000ABF0, 0x0000ABF9}, {0x0000FF10, 0x0000FF19}, {0x00010107, 0x00010133}, -{0x00010140, 0x00010178}, {0x0001018A, 0x0001018B}, {0x000102E1, 0x000102FB}, {0x00010320, 0x00010323}, -{0x00010341, 0x00010341}, {0x0001034A, 0x0001034A}, {0x000103D1, 0x000103D5}, {0x000104A0, 0x000104A9}, -{0x00010858, 0x0001085F}, {0x00010879, 0x0001087F}, {0x000108A7, 0x000108AF}, {0x000108FB, 0x000108FF}, -{0x00010916, 0x0001091B}, {0x000109BC, 0x000109BD}, {0x000109C0, 0x000109CF}, {0x000109D2, 0x000109FF}, -{0x00010A40, 0x00010A48}, {0x00010A7D, 0x00010A7E}, {0x00010A9D, 0x00010A9F}, {0x00010AEB, 0x00010AEF}, -{0x00010B58, 0x00010B5F}, {0x00010B78, 0x00010B7F}, {0x00010BA9, 0x00010BAF}, {0x00010CFA, 0x00010CFF}, -{0x00010D30, 0x00010D39}, {0x00010E60, 0x00010E7E}, {0x00010F1D, 0x00010F26}, {0x00010F51, 0x00010F54}, -{0x00010FC5, 0x00010FCB}, {0x00011052, 0x0001106F}, {0x000110F0, 0x000110F9}, {0x00011136, 0x0001113F}, -{0x000111D0, 0x000111D9}, {0x000111E1, 0x000111F4}, {0x000112F0, 0x000112F9}, {0x00011450, 0x00011459}, -{0x000114D0, 0x000114D9}, {0x00011650, 0x00011659}, {0x000116C0, 0x000116C9}, {0x00011730, 0x0001173B}, -{0x000118E0, 0x000118F2}, {0x00011950, 0x00011959}, {0x00011C50, 0x00011C6C}, {0x00011D50, 0x00011D59}, -{0x00011DA0, 0x00011DA9}, {0x00011F50, 0x00011F59}, {0x00011FC0, 0x00011FD4}, {0x00012400, 0x0001246E}, -{0x00016A60, 0x00016A69}, {0x00016AC0, 0x00016AC9}, {0x00016B50, 0x00016B59}, {0x00016B5B, 0x00016B61}, -{0x00016E80, 0x00016E96}, {0x0001D2C0, 0x0001D2D3}, {0x0001D2E0, 0x0001D2F3}, {0x0001D360, 0x0001D378}, -{0x0001D7CE, 0x0001D7FF}, {0x0001E140, 0x0001E149}, {0x0001E2F0, 0x0001E2F9}, {0x0001E4F0, 0x0001E4F9}, -{0x0001E8C7, 0x0001E8CF}, {0x0001E950, 0x0001E959}, {0x0001EC71, 0x0001ECAB}, {0x0001ECAD, 0x0001ECAF}, -{0x0001ECB1, 0x0001ECB4}, {0x0001ED01, 0x0001ED2D}, {0x0001ED2F, 0x0001ED3D}, {0x0001F100, 0x0001F10C}, -{0x0001FBF0, 0x0001FBF9}, +const std::vector> unicode_ranges_flags = { // start, flags // last=next_start-1 +{0x000000, 0x0080}, +{0x000020, 0x0008}, +{0x000021, 0x0020}, +{0x000024, 0x0040}, +{0x000025, 0x0020}, +{0x00002B, 0x0040}, +{0x00002C, 0x0020}, +{0x000030, 0x0002}, +{0x00003A, 0x0020}, +{0x00003C, 0x0040}, +{0x00003F, 0x0020}, +{0x000041, 0x0004}, +{0x00005B, 0x0020}, +{0x00005E, 0x0040}, +{0x00005F, 0x0020}, +{0x000060, 0x0040}, +{0x000061, 0x0004}, +{0x00007B, 0x0020}, +{0x00007C, 0x0040}, +{0x00007D, 0x0020}, +{0x00007E, 0x0040}, +{0x00007F, 0x0080}, +{0x0000A0, 0x0008}, +{0x0000A1, 0x0020}, +{0x0000A2, 0x0040}, +{0x0000A7, 0x0020}, +{0x0000A8, 0x0040}, +{0x0000AA, 0x0004}, +{0x0000AB, 0x0020}, +{0x0000AC, 0x0040}, +{0x0000AD, 0x0080}, +{0x0000AE, 0x0040}, +{0x0000B2, 0x0002}, +{0x0000B4, 0x0040}, +{0x0000B5, 0x0004}, +{0x0000B6, 0x0020}, +{0x0000B8, 0x0040}, +{0x0000B9, 0x0002}, +{0x0000BA, 0x0004}, +{0x0000BB, 0x0020}, +{0x0000BC, 0x0002}, +{0x0000BF, 0x0020}, +{0x0000C0, 0x0004}, +{0x0000D7, 0x0040}, +{0x0000D8, 0x0004}, +{0x0000F7, 0x0040}, +{0x0000F8, 0x0004}, +{0x0002C2, 0x0040}, +{0x0002C6, 0x0004}, +{0x0002D2, 0x0040}, +{0x0002E0, 0x0004}, +{0x0002E5, 0x0040}, +{0x0002EC, 0x0004}, +{0x0002ED, 0x0040}, +{0x0002EE, 0x0004}, +{0x0002EF, 0x0040}, +{0x000300, 0x0010}, +{0x000370, 0x0004}, +{0x000375, 0x0040}, +{0x000376, 0x0004}, +{0x000378, 0x0080}, +{0x00037A, 0x0004}, +{0x00037E, 0x0020}, +{0x00037F, 0x0004}, +{0x000380, 0x0080}, +{0x000384, 0x0040}, +{0x000386, 0x0004}, +{0x000387, 0x0020}, +{0x000388, 0x0004}, +{0x00038B, 0x0080}, +{0x00038C, 0x0004}, +{0x00038D, 0x0080}, +{0x00038E, 0x0004}, +{0x0003A2, 0x0080}, +{0x0003A3, 0x0004}, +{0x0003F6, 0x0040}, +{0x0003F7, 0x0004}, +{0x000482, 0x0040}, +{0x000483, 0x0010}, +{0x00048A, 0x0004}, +{0x000530, 0x0080}, +{0x000531, 0x0004}, +{0x000557, 0x0080}, +{0x000559, 0x0004}, +{0x00055A, 0x0020}, +{0x000560, 0x0004}, +{0x000589, 0x0020}, +{0x00058B, 0x0080}, +{0x00058D, 0x0040}, +{0x000590, 0x0080}, +{0x000591, 0x0010}, +{0x0005BE, 0x0020}, +{0x0005BF, 0x0010}, +{0x0005C0, 0x0020}, +{0x0005C1, 0x0010}, +{0x0005C3, 0x0020}, +{0x0005C4, 0x0010}, +{0x0005C6, 0x0020}, +{0x0005C7, 0x0010}, +{0x0005C8, 0x0080}, +{0x0005D0, 0x0004}, +{0x0005EB, 0x0080}, +{0x0005EF, 0x0004}, +{0x0005F3, 0x0020}, +{0x0005F5, 0x0080}, +{0x000606, 0x0040}, +{0x000609, 0x0020}, +{0x00060B, 0x0040}, +{0x00060C, 0x0020}, +{0x00060E, 0x0040}, +{0x000610, 0x0010}, +{0x00061B, 0x0020}, +{0x00061C, 0x0080}, +{0x00061D, 0x0020}, +{0x000620, 0x0004}, +{0x00064B, 0x0010}, +{0x000660, 0x0002}, +{0x00066A, 0x0020}, +{0x00066E, 0x0004}, +{0x000670, 0x0010}, +{0x000671, 0x0004}, +{0x0006D4, 0x0020}, +{0x0006D5, 0x0004}, +{0x0006D6, 0x0010}, +{0x0006DD, 0x0080}, +{0x0006DE, 0x0040}, +{0x0006DF, 0x0010}, +{0x0006E5, 0x0004}, +{0x0006E7, 0x0010}, +{0x0006E9, 0x0040}, +{0x0006EA, 0x0010}, +{0x0006EE, 0x0004}, +{0x0006F0, 0x0002}, +{0x0006FA, 0x0004}, +{0x0006FD, 0x0040}, +{0x0006FF, 0x0004}, +{0x000700, 0x0020}, +{0x00070E, 0x0080}, +{0x000710, 0x0004}, +{0x000711, 0x0010}, +{0x000712, 0x0004}, +{0x000730, 0x0010}, +{0x00074B, 0x0080}, +{0x00074D, 0x0004}, +{0x0007A6, 0x0010}, +{0x0007B1, 0x0004}, +{0x0007B2, 0x0080}, +{0x0007C0, 0x0002}, +{0x0007CA, 0x0004}, +{0x0007EB, 0x0010}, +{0x0007F4, 0x0004}, +{0x0007F6, 0x0040}, +{0x0007F7, 0x0020}, +{0x0007FA, 0x0004}, +{0x0007FB, 0x0080}, +{0x0007FD, 0x0010}, +{0x0007FE, 0x0040}, +{0x000800, 0x0004}, +{0x000816, 0x0010}, +{0x00081A, 0x0004}, +{0x00081B, 0x0010}, +{0x000824, 0x0004}, +{0x000825, 0x0010}, +{0x000828, 0x0004}, +{0x000829, 0x0010}, +{0x00082E, 0x0080}, +{0x000830, 0x0020}, +{0x00083F, 0x0080}, +{0x000840, 0x0004}, +{0x000859, 0x0010}, +{0x00085C, 0x0080}, +{0x00085E, 0x0020}, +{0x00085F, 0x0080}, +{0x000860, 0x0004}, +{0x00086B, 0x0080}, +{0x000870, 0x0004}, +{0x000888, 0x0040}, +{0x000889, 0x0004}, +{0x00088F, 0x0080}, +{0x000898, 0x0010}, +{0x0008A0, 0x0004}, +{0x0008CA, 0x0010}, +{0x0008E2, 0x0080}, +{0x0008E3, 0x0010}, +{0x000904, 0x0004}, +{0x00093A, 0x0010}, +{0x00093D, 0x0004}, +{0x00093E, 0x0010}, +{0x000950, 0x0004}, +{0x000951, 0x0010}, +{0x000958, 0x0004}, +{0x000962, 0x0010}, +{0x000964, 0x0020}, +{0x000966, 0x0002}, +{0x000970, 0x0020}, +{0x000971, 0x0004}, +{0x000981, 0x0010}, +{0x000984, 0x0080}, +{0x000985, 0x0004}, +{0x00098D, 0x0080}, +{0x00098F, 0x0004}, +{0x000991, 0x0080}, +{0x000993, 0x0004}, +{0x0009A9, 0x0080}, +{0x0009AA, 0x0004}, +{0x0009B1, 0x0080}, +{0x0009B2, 0x0004}, +{0x0009B3, 0x0080}, +{0x0009B6, 0x0004}, +{0x0009BA, 0x0080}, +{0x0009BC, 0x0010}, +{0x0009BD, 0x0004}, +{0x0009BE, 0x0010}, +{0x0009C5, 0x0080}, +{0x0009C7, 0x0010}, +{0x0009C9, 0x0080}, +{0x0009CB, 0x0010}, +{0x0009CE, 0x0004}, +{0x0009CF, 0x0080}, +{0x0009D7, 0x0010}, +{0x0009D8, 0x0080}, +{0x0009DC, 0x0004}, +{0x0009DE, 0x0080}, +{0x0009DF, 0x0004}, +{0x0009E2, 0x0010}, +{0x0009E4, 0x0080}, +{0x0009E6, 0x0002}, +{0x0009F0, 0x0004}, +{0x0009F2, 0x0040}, +{0x0009F4, 0x0002}, +{0x0009FA, 0x0040}, +{0x0009FC, 0x0004}, +{0x0009FD, 0x0020}, +{0x0009FE, 0x0010}, +{0x0009FF, 0x0080}, +{0x000A01, 0x0010}, +{0x000A04, 0x0080}, +{0x000A05, 0x0004}, +{0x000A0B, 0x0080}, +{0x000A0F, 0x0004}, +{0x000A11, 0x0080}, +{0x000A13, 0x0004}, +{0x000A29, 0x0080}, +{0x000A2A, 0x0004}, +{0x000A31, 0x0080}, +{0x000A32, 0x0004}, +{0x000A34, 0x0080}, +{0x000A35, 0x0004}, +{0x000A37, 0x0080}, +{0x000A38, 0x0004}, +{0x000A3A, 0x0080}, +{0x000A3C, 0x0010}, +{0x000A3D, 0x0080}, +{0x000A3E, 0x0010}, +{0x000A43, 0x0080}, +{0x000A47, 0x0010}, +{0x000A49, 0x0080}, +{0x000A4B, 0x0010}, +{0x000A4E, 0x0080}, +{0x000A51, 0x0010}, +{0x000A52, 0x0080}, +{0x000A59, 0x0004}, +{0x000A5D, 0x0080}, +{0x000A5E, 0x0004}, +{0x000A5F, 0x0080}, +{0x000A66, 0x0002}, +{0x000A70, 0x0010}, +{0x000A72, 0x0004}, +{0x000A75, 0x0010}, +{0x000A76, 0x0020}, +{0x000A77, 0x0080}, +{0x000A81, 0x0010}, +{0x000A84, 0x0080}, +{0x000A85, 0x0004}, +{0x000A8E, 0x0080}, +{0x000A8F, 0x0004}, +{0x000A92, 0x0080}, +{0x000A93, 0x0004}, +{0x000AA9, 0x0080}, +{0x000AAA, 0x0004}, +{0x000AB1, 0x0080}, +{0x000AB2, 0x0004}, +{0x000AB4, 0x0080}, +{0x000AB5, 0x0004}, +{0x000ABA, 0x0080}, +{0x000ABC, 0x0010}, +{0x000ABD, 0x0004}, +{0x000ABE, 0x0010}, +{0x000AC6, 0x0080}, +{0x000AC7, 0x0010}, +{0x000ACA, 0x0080}, +{0x000ACB, 0x0010}, +{0x000ACE, 0x0080}, +{0x000AD0, 0x0004}, +{0x000AD1, 0x0080}, +{0x000AE0, 0x0004}, +{0x000AE2, 0x0010}, +{0x000AE4, 0x0080}, +{0x000AE6, 0x0002}, +{0x000AF0, 0x0020}, +{0x000AF1, 0x0040}, +{0x000AF2, 0x0080}, +{0x000AF9, 0x0004}, +{0x000AFA, 0x0010}, +{0x000B00, 0x0080}, +{0x000B01, 0x0010}, +{0x000B04, 0x0080}, +{0x000B05, 0x0004}, +{0x000B0D, 0x0080}, +{0x000B0F, 0x0004}, +{0x000B11, 0x0080}, +{0x000B13, 0x0004}, +{0x000B29, 0x0080}, +{0x000B2A, 0x0004}, +{0x000B31, 0x0080}, +{0x000B32, 0x0004}, +{0x000B34, 0x0080}, +{0x000B35, 0x0004}, +{0x000B3A, 0x0080}, +{0x000B3C, 0x0010}, +{0x000B3D, 0x0004}, +{0x000B3E, 0x0010}, +{0x000B45, 0x0080}, +{0x000B47, 0x0010}, +{0x000B49, 0x0080}, +{0x000B4B, 0x0010}, +{0x000B4E, 0x0080}, +{0x000B55, 0x0010}, +{0x000B58, 0x0080}, +{0x000B5C, 0x0004}, +{0x000B5E, 0x0080}, +{0x000B5F, 0x0004}, +{0x000B62, 0x0010}, +{0x000B64, 0x0080}, +{0x000B66, 0x0002}, +{0x000B70, 0x0040}, +{0x000B71, 0x0004}, +{0x000B72, 0x0002}, +{0x000B78, 0x0080}, +{0x000B82, 0x0010}, +{0x000B83, 0x0004}, +{0x000B84, 0x0080}, +{0x000B85, 0x0004}, +{0x000B8B, 0x0080}, +{0x000B8E, 0x0004}, +{0x000B91, 0x0080}, +{0x000B92, 0x0004}, +{0x000B96, 0x0080}, +{0x000B99, 0x0004}, +{0x000B9B, 0x0080}, +{0x000B9C, 0x0004}, +{0x000B9D, 0x0080}, +{0x000B9E, 0x0004}, +{0x000BA0, 0x0080}, +{0x000BA3, 0x0004}, +{0x000BA5, 0x0080}, +{0x000BA8, 0x0004}, +{0x000BAB, 0x0080}, +{0x000BAE, 0x0004}, +{0x000BBA, 0x0080}, +{0x000BBE, 0x0010}, +{0x000BC3, 0x0080}, +{0x000BC6, 0x0010}, +{0x000BC9, 0x0080}, +{0x000BCA, 0x0010}, +{0x000BCE, 0x0080}, +{0x000BD0, 0x0004}, +{0x000BD1, 0x0080}, +{0x000BD7, 0x0010}, +{0x000BD8, 0x0080}, +{0x000BE6, 0x0002}, +{0x000BF3, 0x0040}, +{0x000BFB, 0x0080}, +{0x000C00, 0x0010}, +{0x000C05, 0x0004}, +{0x000C0D, 0x0080}, +{0x000C0E, 0x0004}, +{0x000C11, 0x0080}, +{0x000C12, 0x0004}, +{0x000C29, 0x0080}, +{0x000C2A, 0x0004}, +{0x000C3A, 0x0080}, +{0x000C3C, 0x0010}, +{0x000C3D, 0x0004}, +{0x000C3E, 0x0010}, +{0x000C45, 0x0080}, +{0x000C46, 0x0010}, +{0x000C49, 0x0080}, +{0x000C4A, 0x0010}, +{0x000C4E, 0x0080}, +{0x000C55, 0x0010}, +{0x000C57, 0x0080}, +{0x000C58, 0x0004}, +{0x000C5B, 0x0080}, +{0x000C5D, 0x0004}, +{0x000C5E, 0x0080}, +{0x000C60, 0x0004}, +{0x000C62, 0x0010}, +{0x000C64, 0x0080}, +{0x000C66, 0x0002}, +{0x000C70, 0x0080}, +{0x000C77, 0x0020}, +{0x000C78, 0x0002}, +{0x000C7F, 0x0040}, +{0x000C80, 0x0004}, +{0x000C81, 0x0010}, +{0x000C84, 0x0020}, +{0x000C85, 0x0004}, +{0x000C8D, 0x0080}, +{0x000C8E, 0x0004}, +{0x000C91, 0x0080}, +{0x000C92, 0x0004}, +{0x000CA9, 0x0080}, +{0x000CAA, 0x0004}, +{0x000CB4, 0x0080}, +{0x000CB5, 0x0004}, +{0x000CBA, 0x0080}, +{0x000CBC, 0x0010}, +{0x000CBD, 0x0004}, +{0x000CBE, 0x0010}, +{0x000CC5, 0x0080}, +{0x000CC6, 0x0010}, +{0x000CC9, 0x0080}, +{0x000CCA, 0x0010}, +{0x000CCE, 0x0080}, +{0x000CD5, 0x0010}, +{0x000CD7, 0x0080}, +{0x000CDD, 0x0004}, +{0x000CDF, 0x0080}, +{0x000CE0, 0x0004}, +{0x000CE2, 0x0010}, +{0x000CE4, 0x0080}, +{0x000CE6, 0x0002}, +{0x000CF0, 0x0080}, +{0x000CF1, 0x0004}, +{0x000CF3, 0x0010}, +{0x000CF4, 0x0080}, +{0x000D00, 0x0010}, +{0x000D04, 0x0004}, +{0x000D0D, 0x0080}, +{0x000D0E, 0x0004}, +{0x000D11, 0x0080}, +{0x000D12, 0x0004}, +{0x000D3B, 0x0010}, +{0x000D3D, 0x0004}, +{0x000D3E, 0x0010}, +{0x000D45, 0x0080}, +{0x000D46, 0x0010}, +{0x000D49, 0x0080}, +{0x000D4A, 0x0010}, +{0x000D4E, 0x0004}, +{0x000D4F, 0x0040}, +{0x000D50, 0x0080}, +{0x000D54, 0x0004}, +{0x000D57, 0x0010}, +{0x000D58, 0x0002}, +{0x000D5F, 0x0004}, +{0x000D62, 0x0010}, +{0x000D64, 0x0080}, +{0x000D66, 0x0002}, +{0x000D79, 0x0040}, +{0x000D7A, 0x0004}, +{0x000D80, 0x0080}, +{0x000D81, 0x0010}, +{0x000D84, 0x0080}, +{0x000D85, 0x0004}, +{0x000D97, 0x0080}, +{0x000D9A, 0x0004}, +{0x000DB2, 0x0080}, +{0x000DB3, 0x0004}, +{0x000DBC, 0x0080}, +{0x000DBD, 0x0004}, +{0x000DBE, 0x0080}, +{0x000DC0, 0x0004}, +{0x000DC7, 0x0080}, +{0x000DCA, 0x0010}, +{0x000DCB, 0x0080}, +{0x000DCF, 0x0010}, +{0x000DD5, 0x0080}, +{0x000DD6, 0x0010}, +{0x000DD7, 0x0080}, +{0x000DD8, 0x0010}, +{0x000DE0, 0x0080}, +{0x000DE6, 0x0002}, +{0x000DF0, 0x0080}, +{0x000DF2, 0x0010}, +{0x000DF4, 0x0020}, +{0x000DF5, 0x0080}, +{0x000E01, 0x0004}, +{0x000E31, 0x0010}, +{0x000E32, 0x0004}, +{0x000E34, 0x0010}, +{0x000E3B, 0x0080}, +{0x000E3F, 0x0040}, +{0x000E40, 0x0004}, +{0x000E47, 0x0010}, +{0x000E4F, 0x0020}, +{0x000E50, 0x0002}, +{0x000E5A, 0x0020}, +{0x000E5C, 0x0080}, +{0x000E81, 0x0004}, +{0x000E83, 0x0080}, +{0x000E84, 0x0004}, +{0x000E85, 0x0080}, +{0x000E86, 0x0004}, +{0x000E8B, 0x0080}, +{0x000E8C, 0x0004}, +{0x000EA4, 0x0080}, +{0x000EA5, 0x0004}, +{0x000EA6, 0x0080}, +{0x000EA7, 0x0004}, +{0x000EB1, 0x0010}, +{0x000EB2, 0x0004}, +{0x000EB4, 0x0010}, +{0x000EBD, 0x0004}, +{0x000EBE, 0x0080}, +{0x000EC0, 0x0004}, +{0x000EC5, 0x0080}, +{0x000EC6, 0x0004}, +{0x000EC7, 0x0080}, +{0x000EC8, 0x0010}, +{0x000ECF, 0x0080}, +{0x000ED0, 0x0002}, +{0x000EDA, 0x0080}, +{0x000EDC, 0x0004}, +{0x000EE0, 0x0080}, +{0x000F00, 0x0004}, +{0x000F01, 0x0040}, +{0x000F04, 0x0020}, +{0x000F13, 0x0040}, +{0x000F14, 0x0020}, +{0x000F15, 0x0040}, +{0x000F18, 0x0010}, +{0x000F1A, 0x0040}, +{0x000F20, 0x0002}, +{0x000F34, 0x0040}, +{0x000F35, 0x0010}, +{0x000F36, 0x0040}, +{0x000F37, 0x0010}, +{0x000F38, 0x0040}, +{0x000F39, 0x0010}, +{0x000F3A, 0x0020}, +{0x000F3E, 0x0010}, +{0x000F40, 0x0004}, +{0x000F48, 0x0080}, +{0x000F49, 0x0004}, +{0x000F6D, 0x0080}, +{0x000F71, 0x0010}, +{0x000F85, 0x0020}, +{0x000F86, 0x0010}, +{0x000F88, 0x0004}, +{0x000F8D, 0x0010}, +{0x000F98, 0x0080}, +{0x000F99, 0x0010}, +{0x000FBD, 0x0080}, +{0x000FBE, 0x0040}, +{0x000FC6, 0x0010}, +{0x000FC7, 0x0040}, +{0x000FCD, 0x0080}, +{0x000FCE, 0x0040}, +{0x000FD0, 0x0020}, +{0x000FD5, 0x0040}, +{0x000FD9, 0x0020}, +{0x000FDB, 0x0080}, +{0x001000, 0x0004}, +{0x00102B, 0x0010}, +{0x00103F, 0x0004}, +{0x001040, 0x0002}, +{0x00104A, 0x0020}, +{0x001050, 0x0004}, +{0x001056, 0x0010}, +{0x00105A, 0x0004}, +{0x00105E, 0x0010}, +{0x001061, 0x0004}, +{0x001062, 0x0010}, +{0x001065, 0x0004}, +{0x001067, 0x0010}, +{0x00106E, 0x0004}, +{0x001071, 0x0010}, +{0x001075, 0x0004}, +{0x001082, 0x0010}, +{0x00108E, 0x0004}, +{0x00108F, 0x0010}, +{0x001090, 0x0002}, +{0x00109A, 0x0010}, +{0x00109E, 0x0040}, +{0x0010A0, 0x0004}, +{0x0010C6, 0x0080}, +{0x0010C7, 0x0004}, +{0x0010C8, 0x0080}, +{0x0010CD, 0x0004}, +{0x0010CE, 0x0080}, +{0x0010D0, 0x0004}, +{0x0010FB, 0x0020}, +{0x0010FC, 0x0004}, +{0x001249, 0x0080}, +{0x00124A, 0x0004}, +{0x00124E, 0x0080}, +{0x001250, 0x0004}, +{0x001257, 0x0080}, +{0x001258, 0x0004}, +{0x001259, 0x0080}, +{0x00125A, 0x0004}, +{0x00125E, 0x0080}, +{0x001260, 0x0004}, +{0x001289, 0x0080}, +{0x00128A, 0x0004}, +{0x00128E, 0x0080}, +{0x001290, 0x0004}, +{0x0012B1, 0x0080}, +{0x0012B2, 0x0004}, +{0x0012B6, 0x0080}, +{0x0012B8, 0x0004}, +{0x0012BF, 0x0080}, +{0x0012C0, 0x0004}, +{0x0012C1, 0x0080}, +{0x0012C2, 0x0004}, +{0x0012C6, 0x0080}, +{0x0012C8, 0x0004}, +{0x0012D7, 0x0080}, +{0x0012D8, 0x0004}, +{0x001311, 0x0080}, +{0x001312, 0x0004}, +{0x001316, 0x0080}, +{0x001318, 0x0004}, +{0x00135B, 0x0080}, +{0x00135D, 0x0010}, +{0x001360, 0x0020}, +{0x001369, 0x0002}, +{0x00137D, 0x0080}, +{0x001380, 0x0004}, +{0x001390, 0x0040}, +{0x00139A, 0x0080}, +{0x0013A0, 0x0004}, +{0x0013F6, 0x0080}, +{0x0013F8, 0x0004}, +{0x0013FE, 0x0080}, +{0x001400, 0x0020}, +{0x001401, 0x0004}, +{0x00166D, 0x0040}, +{0x00166E, 0x0020}, +{0x00166F, 0x0004}, +{0x001680, 0x0008}, +{0x001681, 0x0004}, +{0x00169B, 0x0020}, +{0x00169D, 0x0080}, +{0x0016A0, 0x0004}, +{0x0016EB, 0x0020}, +{0x0016EE, 0x0002}, +{0x0016F1, 0x0004}, +{0x0016F9, 0x0080}, +{0x001700, 0x0004}, +{0x001712, 0x0010}, +{0x001716, 0x0080}, +{0x00171F, 0x0004}, +{0x001732, 0x0010}, +{0x001735, 0x0020}, +{0x001737, 0x0080}, +{0x001740, 0x0004}, +{0x001752, 0x0010}, +{0x001754, 0x0080}, +{0x001760, 0x0004}, +{0x00176D, 0x0080}, +{0x00176E, 0x0004}, +{0x001771, 0x0080}, +{0x001772, 0x0010}, +{0x001774, 0x0080}, +{0x001780, 0x0004}, +{0x0017B4, 0x0010}, +{0x0017D4, 0x0020}, +{0x0017D7, 0x0004}, +{0x0017D8, 0x0020}, +{0x0017DB, 0x0040}, +{0x0017DC, 0x0004}, +{0x0017DD, 0x0010}, +{0x0017DE, 0x0080}, +{0x0017E0, 0x0002}, +{0x0017EA, 0x0080}, +{0x0017F0, 0x0002}, +{0x0017FA, 0x0080}, +{0x001800, 0x0020}, +{0x00180B, 0x0010}, +{0x00180E, 0x0080}, +{0x00180F, 0x0010}, +{0x001810, 0x0002}, +{0x00181A, 0x0080}, +{0x001820, 0x0004}, +{0x001879, 0x0080}, +{0x001880, 0x0004}, +{0x001885, 0x0010}, +{0x001887, 0x0004}, +{0x0018A9, 0x0010}, +{0x0018AA, 0x0004}, +{0x0018AB, 0x0080}, +{0x0018B0, 0x0004}, +{0x0018F6, 0x0080}, +{0x001900, 0x0004}, +{0x00191F, 0x0080}, +{0x001920, 0x0010}, +{0x00192C, 0x0080}, +{0x001930, 0x0010}, +{0x00193C, 0x0080}, +{0x001940, 0x0040}, +{0x001941, 0x0080}, +{0x001944, 0x0020}, +{0x001946, 0x0002}, +{0x001950, 0x0004}, +{0x00196E, 0x0080}, +{0x001970, 0x0004}, +{0x001975, 0x0080}, +{0x001980, 0x0004}, +{0x0019AC, 0x0080}, +{0x0019B0, 0x0004}, +{0x0019CA, 0x0080}, +{0x0019D0, 0x0002}, +{0x0019DB, 0x0080}, +{0x0019DE, 0x0040}, +{0x001A00, 0x0004}, +{0x001A17, 0x0010}, +{0x001A1C, 0x0080}, +{0x001A1E, 0x0020}, +{0x001A20, 0x0004}, +{0x001A55, 0x0010}, +{0x001A5F, 0x0080}, +{0x001A60, 0x0010}, +{0x001A7D, 0x0080}, +{0x001A7F, 0x0010}, +{0x001A80, 0x0002}, +{0x001A8A, 0x0080}, +{0x001A90, 0x0002}, +{0x001A9A, 0x0080}, +{0x001AA0, 0x0020}, +{0x001AA7, 0x0004}, +{0x001AA8, 0x0020}, +{0x001AAE, 0x0080}, +{0x001AB0, 0x0010}, +{0x001ACF, 0x0080}, +{0x001B00, 0x0010}, +{0x001B05, 0x0004}, +{0x001B34, 0x0010}, +{0x001B45, 0x0004}, +{0x001B4D, 0x0080}, +{0x001B50, 0x0002}, +{0x001B5A, 0x0020}, +{0x001B61, 0x0040}, +{0x001B6B, 0x0010}, +{0x001B74, 0x0040}, +{0x001B7D, 0x0020}, +{0x001B7F, 0x0080}, +{0x001B80, 0x0010}, +{0x001B83, 0x0004}, +{0x001BA1, 0x0010}, +{0x001BAE, 0x0004}, +{0x001BB0, 0x0002}, +{0x001BBA, 0x0004}, +{0x001BE6, 0x0010}, +{0x001BF4, 0x0080}, +{0x001BFC, 0x0020}, +{0x001C00, 0x0004}, +{0x001C24, 0x0010}, +{0x001C38, 0x0080}, +{0x001C3B, 0x0020}, +{0x001C40, 0x0002}, +{0x001C4A, 0x0080}, +{0x001C4D, 0x0004}, +{0x001C50, 0x0002}, +{0x001C5A, 0x0004}, +{0x001C7E, 0x0020}, +{0x001C80, 0x0004}, +{0x001C89, 0x0080}, +{0x001C90, 0x0004}, +{0x001CBB, 0x0080}, +{0x001CBD, 0x0004}, +{0x001CC0, 0x0020}, +{0x001CC8, 0x0080}, +{0x001CD0, 0x0010}, +{0x001CD3, 0x0020}, +{0x001CD4, 0x0010}, +{0x001CE9, 0x0004}, +{0x001CED, 0x0010}, +{0x001CEE, 0x0004}, +{0x001CF4, 0x0010}, +{0x001CF5, 0x0004}, +{0x001CF7, 0x0010}, +{0x001CFA, 0x0004}, +{0x001CFB, 0x0080}, +{0x001D00, 0x0004}, +{0x001DC0, 0x0010}, +{0x001E00, 0x0004}, +{0x001F16, 0x0080}, +{0x001F18, 0x0004}, +{0x001F1E, 0x0080}, +{0x001F20, 0x0004}, +{0x001F46, 0x0080}, +{0x001F48, 0x0004}, +{0x001F4E, 0x0080}, +{0x001F50, 0x0004}, +{0x001F58, 0x0080}, +{0x001F59, 0x0004}, +{0x001F5A, 0x0080}, +{0x001F5B, 0x0004}, +{0x001F5C, 0x0080}, +{0x001F5D, 0x0004}, +{0x001F5E, 0x0080}, +{0x001F5F, 0x0004}, +{0x001F7E, 0x0080}, +{0x001F80, 0x0004}, +{0x001FB5, 0x0080}, +{0x001FB6, 0x0004}, +{0x001FBD, 0x0040}, +{0x001FBE, 0x0004}, +{0x001FBF, 0x0040}, +{0x001FC2, 0x0004}, +{0x001FC5, 0x0080}, +{0x001FC6, 0x0004}, +{0x001FCD, 0x0040}, +{0x001FD0, 0x0004}, +{0x001FD4, 0x0080}, +{0x001FD6, 0x0004}, +{0x001FDC, 0x0080}, +{0x001FDD, 0x0040}, +{0x001FE0, 0x0004}, +{0x001FED, 0x0040}, +{0x001FF0, 0x0080}, +{0x001FF2, 0x0004}, +{0x001FF5, 0x0080}, +{0x001FF6, 0x0004}, +{0x001FFD, 0x0040}, +{0x001FFF, 0x0080}, +{0x002000, 0x0008}, +{0x00200B, 0x0080}, +{0x002010, 0x0020}, +{0x002028, 0x0008}, +{0x00202A, 0x0080}, +{0x00202F, 0x0008}, +{0x002030, 0x0020}, +{0x002044, 0x0040}, +{0x002045, 0x0020}, +{0x002052, 0x0040}, +{0x002053, 0x0020}, +{0x00205F, 0x0008}, +{0x002060, 0x0080}, +{0x002070, 0x0002}, +{0x002071, 0x0004}, +{0x002072, 0x0080}, +{0x002074, 0x0002}, +{0x00207A, 0x0040}, +{0x00207D, 0x0020}, +{0x00207F, 0x0004}, +{0x002080, 0x0002}, +{0x00208A, 0x0040}, +{0x00208D, 0x0020}, +{0x00208F, 0x0080}, +{0x002090, 0x0004}, +{0x00209D, 0x0080}, +{0x0020A0, 0x0040}, +{0x0020C1, 0x0080}, +{0x0020D0, 0x0010}, +{0x0020F1, 0x0080}, +{0x002100, 0x0040}, +{0x002102, 0x0004}, +{0x002103, 0x0040}, +{0x002107, 0x0004}, +{0x002108, 0x0040}, +{0x00210A, 0x0004}, +{0x002114, 0x0040}, +{0x002115, 0x0004}, +{0x002116, 0x0040}, +{0x002119, 0x0004}, +{0x00211E, 0x0040}, +{0x002124, 0x0004}, +{0x002125, 0x0040}, +{0x002126, 0x0004}, +{0x002127, 0x0040}, +{0x002128, 0x0004}, +{0x002129, 0x0040}, +{0x00212A, 0x0004}, +{0x00212E, 0x0040}, +{0x00212F, 0x0004}, +{0x00213A, 0x0040}, +{0x00213C, 0x0004}, +{0x002140, 0x0040}, +{0x002145, 0x0004}, +{0x00214A, 0x0040}, +{0x00214E, 0x0004}, +{0x00214F, 0x0040}, +{0x002150, 0x0002}, +{0x002183, 0x0004}, +{0x002185, 0x0002}, +{0x00218A, 0x0040}, +{0x00218C, 0x0080}, +{0x002190, 0x0040}, +{0x002308, 0x0020}, +{0x00230C, 0x0040}, +{0x002329, 0x0020}, +{0x00232B, 0x0040}, +{0x002427, 0x0080}, +{0x002440, 0x0040}, +{0x00244B, 0x0080}, +{0x002460, 0x0002}, +{0x00249C, 0x0040}, +{0x0024EA, 0x0002}, +{0x002500, 0x0040}, +{0x002768, 0x0020}, +{0x002776, 0x0002}, +{0x002794, 0x0040}, +{0x0027C5, 0x0020}, +{0x0027C7, 0x0040}, +{0x0027E6, 0x0020}, +{0x0027F0, 0x0040}, +{0x002983, 0x0020}, +{0x002999, 0x0040}, +{0x0029D8, 0x0020}, +{0x0029DC, 0x0040}, +{0x0029FC, 0x0020}, +{0x0029FE, 0x0040}, +{0x002B74, 0x0080}, +{0x002B76, 0x0040}, +{0x002B96, 0x0080}, +{0x002B97, 0x0040}, +{0x002C00, 0x0004}, +{0x002CE5, 0x0040}, +{0x002CEB, 0x0004}, +{0x002CEF, 0x0010}, +{0x002CF2, 0x0004}, +{0x002CF4, 0x0080}, +{0x002CF9, 0x0020}, +{0x002CFD, 0x0002}, +{0x002CFE, 0x0020}, +{0x002D00, 0x0004}, +{0x002D26, 0x0080}, +{0x002D27, 0x0004}, +{0x002D28, 0x0080}, +{0x002D2D, 0x0004}, +{0x002D2E, 0x0080}, +{0x002D30, 0x0004}, +{0x002D68, 0x0080}, +{0x002D6F, 0x0004}, +{0x002D70, 0x0020}, +{0x002D71, 0x0080}, +{0x002D7F, 0x0010}, +{0x002D80, 0x0004}, +{0x002D97, 0x0080}, +{0x002DA0, 0x0004}, +{0x002DA7, 0x0080}, +{0x002DA8, 0x0004}, +{0x002DAF, 0x0080}, +{0x002DB0, 0x0004}, +{0x002DB7, 0x0080}, +{0x002DB8, 0x0004}, +{0x002DBF, 0x0080}, +{0x002DC0, 0x0004}, +{0x002DC7, 0x0080}, +{0x002DC8, 0x0004}, +{0x002DCF, 0x0080}, +{0x002DD0, 0x0004}, +{0x002DD7, 0x0080}, +{0x002DD8, 0x0004}, +{0x002DDF, 0x0080}, +{0x002DE0, 0x0010}, +{0x002E00, 0x0020}, +{0x002E2F, 0x0004}, +{0x002E30, 0x0020}, +{0x002E50, 0x0040}, +{0x002E52, 0x0020}, +{0x002E5E, 0x0080}, +{0x002E80, 0x0040}, +{0x002E9A, 0x0080}, +{0x002E9B, 0x0040}, +{0x002EF4, 0x0080}, +{0x002F00, 0x0040}, +{0x002FD6, 0x0080}, +{0x002FF0, 0x0040}, +{0x003000, 0x0008}, +{0x003001, 0x0020}, +{0x003004, 0x0040}, +{0x003005, 0x0004}, +{0x003007, 0x0002}, +{0x003008, 0x0020}, +{0x003012, 0x0040}, +{0x003014, 0x0020}, +{0x003020, 0x0040}, +{0x003021, 0x0002}, +{0x00302A, 0x0010}, +{0x003030, 0x0020}, +{0x003031, 0x0004}, +{0x003036, 0x0040}, +{0x003038, 0x0002}, +{0x00303B, 0x0004}, +{0x00303D, 0x0020}, +{0x00303E, 0x0040}, +{0x003040, 0x0080}, +{0x003041, 0x0004}, +{0x003097, 0x0080}, +{0x003099, 0x0010}, +{0x00309B, 0x0040}, +{0x00309D, 0x0004}, +{0x0030A0, 0x0020}, +{0x0030A1, 0x0004}, +{0x0030FB, 0x0020}, +{0x0030FC, 0x0004}, +{0x003100, 0x0080}, +{0x003105, 0x0004}, +{0x003130, 0x0080}, +{0x003131, 0x0004}, +{0x00318F, 0x0080}, +{0x003190, 0x0040}, +{0x003192, 0x0002}, +{0x003196, 0x0040}, +{0x0031A0, 0x0004}, +{0x0031C0, 0x0040}, +{0x0031E4, 0x0080}, +{0x0031EF, 0x0040}, +{0x0031F0, 0x0004}, +{0x003200, 0x0040}, +{0x00321F, 0x0080}, +{0x003220, 0x0002}, +{0x00322A, 0x0040}, +{0x003248, 0x0002}, +{0x003250, 0x0040}, +{0x003251, 0x0002}, +{0x003260, 0x0040}, +{0x003280, 0x0002}, +{0x00328A, 0x0040}, +{0x0032B1, 0x0002}, +{0x0032C0, 0x0040}, +{0x003400, 0x0004}, +{0x004DC0, 0x0040}, +{0x004E00, 0x0004}, +{0x00A48D, 0x0080}, +{0x00A490, 0x0040}, +{0x00A4C7, 0x0080}, +{0x00A4D0, 0x0004}, +{0x00A4FE, 0x0020}, +{0x00A500, 0x0004}, +{0x00A60D, 0x0020}, +{0x00A610, 0x0004}, +{0x00A620, 0x0002}, +{0x00A62A, 0x0004}, +{0x00A62C, 0x0080}, +{0x00A640, 0x0004}, +{0x00A66F, 0x0010}, +{0x00A673, 0x0020}, +{0x00A674, 0x0010}, +{0x00A67E, 0x0020}, +{0x00A67F, 0x0004}, +{0x00A69E, 0x0010}, +{0x00A6A0, 0x0004}, +{0x00A6E6, 0x0002}, +{0x00A6F0, 0x0010}, +{0x00A6F2, 0x0020}, +{0x00A6F8, 0x0080}, +{0x00A700, 0x0040}, +{0x00A717, 0x0004}, +{0x00A720, 0x0040}, +{0x00A722, 0x0004}, +{0x00A789, 0x0040}, +{0x00A78B, 0x0004}, +{0x00A7CB, 0x0080}, +{0x00A7D0, 0x0004}, +{0x00A7D2, 0x0080}, +{0x00A7D3, 0x0004}, +{0x00A7D4, 0x0080}, +{0x00A7D5, 0x0004}, +{0x00A7DA, 0x0080}, +{0x00A7F2, 0x0004}, +{0x00A802, 0x0010}, +{0x00A803, 0x0004}, +{0x00A806, 0x0010}, +{0x00A807, 0x0004}, +{0x00A80B, 0x0010}, +{0x00A80C, 0x0004}, +{0x00A823, 0x0010}, +{0x00A828, 0x0040}, +{0x00A82C, 0x0010}, +{0x00A82D, 0x0080}, +{0x00A830, 0x0002}, +{0x00A836, 0x0040}, +{0x00A83A, 0x0080}, +{0x00A840, 0x0004}, +{0x00A874, 0x0020}, +{0x00A878, 0x0080}, +{0x00A880, 0x0010}, +{0x00A882, 0x0004}, +{0x00A8B4, 0x0010}, +{0x00A8C6, 0x0080}, +{0x00A8CE, 0x0020}, +{0x00A8D0, 0x0002}, +{0x00A8DA, 0x0080}, +{0x00A8E0, 0x0010}, +{0x00A8F2, 0x0004}, +{0x00A8F8, 0x0020}, +{0x00A8FB, 0x0004}, +{0x00A8FC, 0x0020}, +{0x00A8FD, 0x0004}, +{0x00A8FF, 0x0010}, +{0x00A900, 0x0002}, +{0x00A90A, 0x0004}, +{0x00A926, 0x0010}, +{0x00A92E, 0x0020}, +{0x00A930, 0x0004}, +{0x00A947, 0x0010}, +{0x00A954, 0x0080}, +{0x00A95F, 0x0020}, +{0x00A960, 0x0004}, +{0x00A97D, 0x0080}, +{0x00A980, 0x0010}, +{0x00A984, 0x0004}, +{0x00A9B3, 0x0010}, +{0x00A9C1, 0x0020}, +{0x00A9CE, 0x0080}, +{0x00A9CF, 0x0004}, +{0x00A9D0, 0x0002}, +{0x00A9DA, 0x0080}, +{0x00A9DE, 0x0020}, +{0x00A9E0, 0x0004}, +{0x00A9E5, 0x0010}, +{0x00A9E6, 0x0004}, +{0x00A9F0, 0x0002}, +{0x00A9FA, 0x0004}, +{0x00A9FF, 0x0080}, +{0x00AA00, 0x0004}, +{0x00AA29, 0x0010}, +{0x00AA37, 0x0080}, +{0x00AA40, 0x0004}, +{0x00AA43, 0x0010}, +{0x00AA44, 0x0004}, +{0x00AA4C, 0x0010}, +{0x00AA4E, 0x0080}, +{0x00AA50, 0x0002}, +{0x00AA5A, 0x0080}, +{0x00AA5C, 0x0020}, +{0x00AA60, 0x0004}, +{0x00AA77, 0x0040}, +{0x00AA7A, 0x0004}, +{0x00AA7B, 0x0010}, +{0x00AA7E, 0x0004}, +{0x00AAB0, 0x0010}, +{0x00AAB1, 0x0004}, +{0x00AAB2, 0x0010}, +{0x00AAB5, 0x0004}, +{0x00AAB7, 0x0010}, +{0x00AAB9, 0x0004}, +{0x00AABE, 0x0010}, +{0x00AAC0, 0x0004}, +{0x00AAC1, 0x0010}, +{0x00AAC2, 0x0004}, +{0x00AAC3, 0x0080}, +{0x00AADB, 0x0004}, +{0x00AADE, 0x0020}, +{0x00AAE0, 0x0004}, +{0x00AAEB, 0x0010}, +{0x00AAF0, 0x0020}, +{0x00AAF2, 0x0004}, +{0x00AAF5, 0x0010}, +{0x00AAF7, 0x0080}, +{0x00AB01, 0x0004}, +{0x00AB07, 0x0080}, +{0x00AB09, 0x0004}, +{0x00AB0F, 0x0080}, +{0x00AB11, 0x0004}, +{0x00AB17, 0x0080}, +{0x00AB20, 0x0004}, +{0x00AB27, 0x0080}, +{0x00AB28, 0x0004}, +{0x00AB2F, 0x0080}, +{0x00AB30, 0x0004}, +{0x00AB5B, 0x0040}, +{0x00AB5C, 0x0004}, +{0x00AB6A, 0x0040}, +{0x00AB6C, 0x0080}, +{0x00AB70, 0x0004}, +{0x00ABE3, 0x0010}, +{0x00ABEB, 0x0020}, +{0x00ABEC, 0x0010}, +{0x00ABEE, 0x0080}, +{0x00ABF0, 0x0002}, +{0x00ABFA, 0x0080}, +{0x00AC00, 0x0004}, +{0x00D7A4, 0x0080}, +{0x00D7B0, 0x0004}, +{0x00D7C7, 0x0080}, +{0x00D7CB, 0x0004}, +{0x00D7FC, 0x0080}, +{0x00F900, 0x0004}, +{0x00FA6E, 0x0080}, +{0x00FA70, 0x0004}, +{0x00FADA, 0x0080}, +{0x00FB00, 0x0004}, +{0x00FB07, 0x0080}, +{0x00FB13, 0x0004}, +{0x00FB18, 0x0080}, +{0x00FB1D, 0x0004}, +{0x00FB1E, 0x0010}, +{0x00FB1F, 0x0004}, +{0x00FB29, 0x0040}, +{0x00FB2A, 0x0004}, +{0x00FB37, 0x0080}, +{0x00FB38, 0x0004}, +{0x00FB3D, 0x0080}, +{0x00FB3E, 0x0004}, +{0x00FB3F, 0x0080}, +{0x00FB40, 0x0004}, +{0x00FB42, 0x0080}, +{0x00FB43, 0x0004}, +{0x00FB45, 0x0080}, +{0x00FB46, 0x0004}, +{0x00FBB2, 0x0040}, +{0x00FBC3, 0x0080}, +{0x00FBD3, 0x0004}, +{0x00FD3E, 0x0020}, +{0x00FD40, 0x0040}, +{0x00FD50, 0x0004}, +{0x00FD90, 0x0080}, +{0x00FD92, 0x0004}, +{0x00FDC8, 0x0080}, +{0x00FDCF, 0x0040}, +{0x00FDD0, 0x0080}, +{0x00FDF0, 0x0004}, +{0x00FDFC, 0x0040}, +{0x00FE00, 0x0010}, +{0x00FE10, 0x0020}, +{0x00FE1A, 0x0080}, +{0x00FE20, 0x0010}, +{0x00FE30, 0x0020}, +{0x00FE53, 0x0080}, +{0x00FE54, 0x0020}, +{0x00FE62, 0x0040}, +{0x00FE63, 0x0020}, +{0x00FE64, 0x0040}, +{0x00FE67, 0x0080}, +{0x00FE68, 0x0020}, +{0x00FE69, 0x0040}, +{0x00FE6A, 0x0020}, +{0x00FE6C, 0x0080}, +{0x00FE70, 0x0004}, +{0x00FE75, 0x0080}, +{0x00FE76, 0x0004}, +{0x00FEFD, 0x0080}, +{0x00FF01, 0x0020}, +{0x00FF04, 0x0040}, +{0x00FF05, 0x0020}, +{0x00FF0B, 0x0040}, +{0x00FF0C, 0x0020}, +{0x00FF10, 0x0002}, +{0x00FF1A, 0x0020}, +{0x00FF1C, 0x0040}, +{0x00FF1F, 0x0020}, +{0x00FF21, 0x0004}, +{0x00FF3B, 0x0020}, +{0x00FF3E, 0x0040}, +{0x00FF3F, 0x0020}, +{0x00FF40, 0x0040}, +{0x00FF41, 0x0004}, +{0x00FF5B, 0x0020}, +{0x00FF5C, 0x0040}, +{0x00FF5D, 0x0020}, +{0x00FF5E, 0x0040}, +{0x00FF5F, 0x0020}, +{0x00FF66, 0x0004}, +{0x00FFBF, 0x0080}, +{0x00FFC2, 0x0004}, +{0x00FFC8, 0x0080}, +{0x00FFCA, 0x0004}, +{0x00FFD0, 0x0080}, +{0x00FFD2, 0x0004}, +{0x00FFD8, 0x0080}, +{0x00FFDA, 0x0004}, +{0x00FFDD, 0x0080}, +{0x00FFE0, 0x0040}, +{0x00FFE7, 0x0080}, +{0x00FFE8, 0x0040}, +{0x00FFEF, 0x0080}, +{0x00FFFC, 0x0040}, +{0x00FFFE, 0x0080}, +{0x010000, 0x0004}, +{0x01000C, 0x0080}, +{0x01000D, 0x0004}, +{0x010027, 0x0080}, +{0x010028, 0x0004}, +{0x01003B, 0x0080}, +{0x01003C, 0x0004}, +{0x01003E, 0x0080}, +{0x01003F, 0x0004}, +{0x01004E, 0x0080}, +{0x010050, 0x0004}, +{0x01005E, 0x0080}, +{0x010080, 0x0004}, +{0x0100FB, 0x0080}, +{0x010100, 0x0020}, +{0x010103, 0x0080}, +{0x010107, 0x0002}, +{0x010134, 0x0080}, +{0x010137, 0x0040}, +{0x010140, 0x0002}, +{0x010179, 0x0040}, +{0x01018A, 0x0002}, +{0x01018C, 0x0040}, +{0x01018F, 0x0080}, +{0x010190, 0x0040}, +{0x01019D, 0x0080}, +{0x0101A0, 0x0040}, +{0x0101A1, 0x0080}, +{0x0101D0, 0x0040}, +{0x0101FD, 0x0010}, +{0x0101FE, 0x0080}, +{0x010280, 0x0004}, +{0x01029D, 0x0080}, +{0x0102A0, 0x0004}, +{0x0102D1, 0x0080}, +{0x0102E0, 0x0010}, +{0x0102E1, 0x0002}, +{0x0102FC, 0x0080}, +{0x010300, 0x0004}, +{0x010320, 0x0002}, +{0x010324, 0x0080}, +{0x01032D, 0x0004}, +{0x010341, 0x0002}, +{0x010342, 0x0004}, +{0x01034A, 0x0002}, +{0x01034B, 0x0080}, +{0x010350, 0x0004}, +{0x010376, 0x0010}, +{0x01037B, 0x0080}, +{0x010380, 0x0004}, +{0x01039E, 0x0080}, +{0x01039F, 0x0020}, +{0x0103A0, 0x0004}, +{0x0103C4, 0x0080}, +{0x0103C8, 0x0004}, +{0x0103D0, 0x0020}, +{0x0103D1, 0x0002}, +{0x0103D6, 0x0080}, +{0x010400, 0x0004}, +{0x01049E, 0x0080}, +{0x0104A0, 0x0002}, +{0x0104AA, 0x0080}, +{0x0104B0, 0x0004}, +{0x0104D4, 0x0080}, +{0x0104D8, 0x0004}, +{0x0104FC, 0x0080}, +{0x010500, 0x0004}, +{0x010528, 0x0080}, +{0x010530, 0x0004}, +{0x010564, 0x0080}, +{0x01056F, 0x0020}, +{0x010570, 0x0004}, +{0x01057B, 0x0080}, +{0x01057C, 0x0004}, +{0x01058B, 0x0080}, +{0x01058C, 0x0004}, +{0x010593, 0x0080}, +{0x010594, 0x0004}, +{0x010596, 0x0080}, +{0x010597, 0x0004}, +{0x0105A2, 0x0080}, +{0x0105A3, 0x0004}, +{0x0105B2, 0x0080}, +{0x0105B3, 0x0004}, +{0x0105BA, 0x0080}, +{0x0105BB, 0x0004}, +{0x0105BD, 0x0080}, +{0x010600, 0x0004}, +{0x010737, 0x0080}, +{0x010740, 0x0004}, +{0x010756, 0x0080}, +{0x010760, 0x0004}, +{0x010768, 0x0080}, +{0x010780, 0x0004}, +{0x010786, 0x0080}, +{0x010787, 0x0004}, +{0x0107B1, 0x0080}, +{0x0107B2, 0x0004}, +{0x0107BB, 0x0080}, +{0x010800, 0x0004}, +{0x010806, 0x0080}, +{0x010808, 0x0004}, +{0x010809, 0x0080}, +{0x01080A, 0x0004}, +{0x010836, 0x0080}, +{0x010837, 0x0004}, +{0x010839, 0x0080}, +{0x01083C, 0x0004}, +{0x01083D, 0x0080}, +{0x01083F, 0x0004}, +{0x010856, 0x0080}, +{0x010857, 0x0020}, +{0x010858, 0x0002}, +{0x010860, 0x0004}, +{0x010877, 0x0040}, +{0x010879, 0x0002}, +{0x010880, 0x0004}, +{0x01089F, 0x0080}, +{0x0108A7, 0x0002}, +{0x0108B0, 0x0080}, +{0x0108E0, 0x0004}, +{0x0108F3, 0x0080}, +{0x0108F4, 0x0004}, +{0x0108F6, 0x0080}, +{0x0108FB, 0x0002}, +{0x010900, 0x0004}, +{0x010916, 0x0002}, +{0x01091C, 0x0080}, +{0x01091F, 0x0020}, +{0x010920, 0x0004}, +{0x01093A, 0x0080}, +{0x01093F, 0x0020}, +{0x010940, 0x0080}, +{0x010980, 0x0004}, +{0x0109B8, 0x0080}, +{0x0109BC, 0x0002}, +{0x0109BE, 0x0004}, +{0x0109C0, 0x0002}, +{0x0109D0, 0x0080}, +{0x0109D2, 0x0002}, +{0x010A00, 0x0004}, +{0x010A01, 0x0010}, +{0x010A04, 0x0080}, +{0x010A05, 0x0010}, +{0x010A07, 0x0080}, +{0x010A0C, 0x0010}, +{0x010A10, 0x0004}, +{0x010A14, 0x0080}, +{0x010A15, 0x0004}, +{0x010A18, 0x0080}, +{0x010A19, 0x0004}, +{0x010A36, 0x0080}, +{0x010A38, 0x0010}, +{0x010A3B, 0x0080}, +{0x010A3F, 0x0010}, +{0x010A40, 0x0002}, +{0x010A49, 0x0080}, +{0x010A50, 0x0020}, +{0x010A59, 0x0080}, +{0x010A60, 0x0004}, +{0x010A7D, 0x0002}, +{0x010A7F, 0x0020}, +{0x010A80, 0x0004}, +{0x010A9D, 0x0002}, +{0x010AA0, 0x0080}, +{0x010AC0, 0x0004}, +{0x010AC8, 0x0040}, +{0x010AC9, 0x0004}, +{0x010AE5, 0x0010}, +{0x010AE7, 0x0080}, +{0x010AEB, 0x0002}, +{0x010AF0, 0x0020}, +{0x010AF7, 0x0080}, +{0x010B00, 0x0004}, +{0x010B36, 0x0080}, +{0x010B39, 0x0020}, +{0x010B40, 0x0004}, +{0x010B56, 0x0080}, +{0x010B58, 0x0002}, +{0x010B60, 0x0004}, +{0x010B73, 0x0080}, +{0x010B78, 0x0002}, +{0x010B80, 0x0004}, +{0x010B92, 0x0080}, +{0x010B99, 0x0020}, +{0x010B9D, 0x0080}, +{0x010BA9, 0x0002}, +{0x010BB0, 0x0080}, +{0x010C00, 0x0004}, +{0x010C49, 0x0080}, +{0x010C80, 0x0004}, +{0x010CB3, 0x0080}, +{0x010CC0, 0x0004}, +{0x010CF3, 0x0080}, +{0x010CFA, 0x0002}, +{0x010D00, 0x0004}, +{0x010D24, 0x0010}, +{0x010D28, 0x0080}, +{0x010D30, 0x0002}, +{0x010D3A, 0x0080}, +{0x010E60, 0x0002}, +{0x010E7F, 0x0080}, +{0x010E80, 0x0004}, +{0x010EAA, 0x0080}, +{0x010EAB, 0x0010}, +{0x010EAD, 0x0020}, +{0x010EAE, 0x0080}, +{0x010EB0, 0x0004}, +{0x010EB2, 0x0080}, +{0x010EFD, 0x0010}, +{0x010F00, 0x0004}, +{0x010F1D, 0x0002}, +{0x010F27, 0x0004}, +{0x010F28, 0x0080}, +{0x010F30, 0x0004}, +{0x010F46, 0x0010}, +{0x010F51, 0x0002}, +{0x010F55, 0x0020}, +{0x010F5A, 0x0080}, +{0x010F70, 0x0004}, +{0x010F82, 0x0010}, +{0x010F86, 0x0020}, +{0x010F8A, 0x0080}, +{0x010FB0, 0x0004}, +{0x010FC5, 0x0002}, +{0x010FCC, 0x0080}, +{0x010FE0, 0x0004}, +{0x010FF7, 0x0080}, +{0x011000, 0x0010}, +{0x011003, 0x0004}, +{0x011038, 0x0010}, +{0x011047, 0x0020}, +{0x01104E, 0x0080}, +{0x011052, 0x0002}, +{0x011070, 0x0010}, +{0x011071, 0x0004}, +{0x011073, 0x0010}, +{0x011075, 0x0004}, +{0x011076, 0x0080}, +{0x01107F, 0x0010}, +{0x011083, 0x0004}, +{0x0110B0, 0x0010}, +{0x0110BB, 0x0020}, +{0x0110BD, 0x0080}, +{0x0110BE, 0x0020}, +{0x0110C2, 0x0010}, +{0x0110C3, 0x0080}, +{0x0110D0, 0x0004}, +{0x0110E9, 0x0080}, +{0x0110F0, 0x0002}, +{0x0110FA, 0x0080}, +{0x011100, 0x0010}, +{0x011103, 0x0004}, +{0x011127, 0x0010}, +{0x011135, 0x0080}, +{0x011136, 0x0002}, +{0x011140, 0x0020}, +{0x011144, 0x0004}, +{0x011145, 0x0010}, +{0x011147, 0x0004}, +{0x011148, 0x0080}, +{0x011150, 0x0004}, +{0x011173, 0x0010}, +{0x011174, 0x0020}, +{0x011176, 0x0004}, +{0x011177, 0x0080}, +{0x011180, 0x0010}, +{0x011183, 0x0004}, +{0x0111B3, 0x0010}, +{0x0111C1, 0x0004}, +{0x0111C5, 0x0020}, +{0x0111C9, 0x0010}, +{0x0111CD, 0x0020}, +{0x0111CE, 0x0010}, +{0x0111D0, 0x0002}, +{0x0111DA, 0x0004}, +{0x0111DB, 0x0020}, +{0x0111DC, 0x0004}, +{0x0111DD, 0x0020}, +{0x0111E0, 0x0080}, +{0x0111E1, 0x0002}, +{0x0111F5, 0x0080}, +{0x011200, 0x0004}, +{0x011212, 0x0080}, +{0x011213, 0x0004}, +{0x01122C, 0x0010}, +{0x011238, 0x0020}, +{0x01123E, 0x0010}, +{0x01123F, 0x0004}, +{0x011241, 0x0010}, +{0x011242, 0x0080}, +{0x011280, 0x0004}, +{0x011287, 0x0080}, +{0x011288, 0x0004}, +{0x011289, 0x0080}, +{0x01128A, 0x0004}, +{0x01128E, 0x0080}, +{0x01128F, 0x0004}, +{0x01129E, 0x0080}, +{0x01129F, 0x0004}, +{0x0112A9, 0x0020}, +{0x0112AA, 0x0080}, +{0x0112B0, 0x0004}, +{0x0112DF, 0x0010}, +{0x0112EB, 0x0080}, +{0x0112F0, 0x0002}, +{0x0112FA, 0x0080}, +{0x011300, 0x0010}, +{0x011304, 0x0080}, +{0x011305, 0x0004}, +{0x01130D, 0x0080}, +{0x01130F, 0x0004}, +{0x011311, 0x0080}, +{0x011313, 0x0004}, +{0x011329, 0x0080}, +{0x01132A, 0x0004}, +{0x011331, 0x0080}, +{0x011332, 0x0004}, +{0x011334, 0x0080}, +{0x011335, 0x0004}, +{0x01133A, 0x0080}, +{0x01133B, 0x0010}, +{0x01133D, 0x0004}, +{0x01133E, 0x0010}, +{0x011345, 0x0080}, +{0x011347, 0x0010}, +{0x011349, 0x0080}, +{0x01134B, 0x0010}, +{0x01134E, 0x0080}, +{0x011350, 0x0004}, +{0x011351, 0x0080}, +{0x011357, 0x0010}, +{0x011358, 0x0080}, +{0x01135D, 0x0004}, +{0x011362, 0x0010}, +{0x011364, 0x0080}, +{0x011366, 0x0010}, +{0x01136D, 0x0080}, +{0x011370, 0x0010}, +{0x011375, 0x0080}, +{0x011400, 0x0004}, +{0x011435, 0x0010}, +{0x011447, 0x0004}, +{0x01144B, 0x0020}, +{0x011450, 0x0002}, +{0x01145A, 0x0020}, +{0x01145C, 0x0080}, +{0x01145D, 0x0020}, +{0x01145E, 0x0010}, +{0x01145F, 0x0004}, +{0x011462, 0x0080}, +{0x011480, 0x0004}, +{0x0114B0, 0x0010}, +{0x0114C4, 0x0004}, +{0x0114C6, 0x0020}, +{0x0114C7, 0x0004}, +{0x0114C8, 0x0080}, +{0x0114D0, 0x0002}, +{0x0114DA, 0x0080}, +{0x011580, 0x0004}, +{0x0115AF, 0x0010}, +{0x0115B6, 0x0080}, +{0x0115B8, 0x0010}, +{0x0115C1, 0x0020}, +{0x0115D8, 0x0004}, +{0x0115DC, 0x0010}, +{0x0115DE, 0x0080}, +{0x011600, 0x0004}, +{0x011630, 0x0010}, +{0x011641, 0x0020}, +{0x011644, 0x0004}, +{0x011645, 0x0080}, +{0x011650, 0x0002}, +{0x01165A, 0x0080}, +{0x011660, 0x0020}, +{0x01166D, 0x0080}, +{0x011680, 0x0004}, +{0x0116AB, 0x0010}, +{0x0116B8, 0x0004}, +{0x0116B9, 0x0020}, +{0x0116BA, 0x0080}, +{0x0116C0, 0x0002}, +{0x0116CA, 0x0080}, +{0x011700, 0x0004}, +{0x01171B, 0x0080}, +{0x01171D, 0x0010}, +{0x01172C, 0x0080}, +{0x011730, 0x0002}, +{0x01173C, 0x0020}, +{0x01173F, 0x0040}, +{0x011740, 0x0004}, +{0x011747, 0x0080}, +{0x011800, 0x0004}, +{0x01182C, 0x0010}, +{0x01183B, 0x0020}, +{0x01183C, 0x0080}, +{0x0118A0, 0x0004}, +{0x0118E0, 0x0002}, +{0x0118F3, 0x0080}, +{0x0118FF, 0x0004}, +{0x011907, 0x0080}, +{0x011909, 0x0004}, +{0x01190A, 0x0080}, +{0x01190C, 0x0004}, +{0x011914, 0x0080}, +{0x011915, 0x0004}, +{0x011917, 0x0080}, +{0x011918, 0x0004}, +{0x011930, 0x0010}, +{0x011936, 0x0080}, +{0x011937, 0x0010}, +{0x011939, 0x0080}, +{0x01193B, 0x0010}, +{0x01193F, 0x0004}, +{0x011940, 0x0010}, +{0x011941, 0x0004}, +{0x011942, 0x0010}, +{0x011944, 0x0020}, +{0x011947, 0x0080}, +{0x011950, 0x0002}, +{0x01195A, 0x0080}, +{0x0119A0, 0x0004}, +{0x0119A8, 0x0080}, +{0x0119AA, 0x0004}, +{0x0119D1, 0x0010}, +{0x0119D8, 0x0080}, +{0x0119DA, 0x0010}, +{0x0119E1, 0x0004}, +{0x0119E2, 0x0020}, +{0x0119E3, 0x0004}, +{0x0119E4, 0x0010}, +{0x0119E5, 0x0080}, +{0x011A00, 0x0004}, +{0x011A01, 0x0010}, +{0x011A0B, 0x0004}, +{0x011A33, 0x0010}, +{0x011A3A, 0x0004}, +{0x011A3B, 0x0010}, +{0x011A3F, 0x0020}, +{0x011A47, 0x0010}, +{0x011A48, 0x0080}, +{0x011A50, 0x0004}, +{0x011A51, 0x0010}, +{0x011A5C, 0x0004}, +{0x011A8A, 0x0010}, +{0x011A9A, 0x0020}, +{0x011A9D, 0x0004}, +{0x011A9E, 0x0020}, +{0x011AA3, 0x0080}, +{0x011AB0, 0x0004}, +{0x011AF9, 0x0080}, +{0x011B00, 0x0020}, +{0x011B0A, 0x0080}, +{0x011C00, 0x0004}, +{0x011C09, 0x0080}, +{0x011C0A, 0x0004}, +{0x011C2F, 0x0010}, +{0x011C37, 0x0080}, +{0x011C38, 0x0010}, +{0x011C40, 0x0004}, +{0x011C41, 0x0020}, +{0x011C46, 0x0080}, +{0x011C50, 0x0002}, +{0x011C6D, 0x0080}, +{0x011C70, 0x0020}, +{0x011C72, 0x0004}, +{0x011C90, 0x0080}, +{0x011C92, 0x0010}, +{0x011CA8, 0x0080}, +{0x011CA9, 0x0010}, +{0x011CB7, 0x0080}, +{0x011D00, 0x0004}, +{0x011D07, 0x0080}, +{0x011D08, 0x0004}, +{0x011D0A, 0x0080}, +{0x011D0B, 0x0004}, +{0x011D31, 0x0010}, +{0x011D37, 0x0080}, +{0x011D3A, 0x0010}, +{0x011D3B, 0x0080}, +{0x011D3C, 0x0010}, +{0x011D3E, 0x0080}, +{0x011D3F, 0x0010}, +{0x011D46, 0x0004}, +{0x011D47, 0x0010}, +{0x011D48, 0x0080}, +{0x011D50, 0x0002}, +{0x011D5A, 0x0080}, +{0x011D60, 0x0004}, +{0x011D66, 0x0080}, +{0x011D67, 0x0004}, +{0x011D69, 0x0080}, +{0x011D6A, 0x0004}, +{0x011D8A, 0x0010}, +{0x011D8F, 0x0080}, +{0x011D90, 0x0010}, +{0x011D92, 0x0080}, +{0x011D93, 0x0010}, +{0x011D98, 0x0004}, +{0x011D99, 0x0080}, +{0x011DA0, 0x0002}, +{0x011DAA, 0x0080}, +{0x011EE0, 0x0004}, +{0x011EF3, 0x0010}, +{0x011EF7, 0x0020}, +{0x011EF9, 0x0080}, +{0x011F00, 0x0010}, +{0x011F02, 0x0004}, +{0x011F03, 0x0010}, +{0x011F04, 0x0004}, +{0x011F11, 0x0080}, +{0x011F12, 0x0004}, +{0x011F34, 0x0010}, +{0x011F3B, 0x0080}, +{0x011F3E, 0x0010}, +{0x011F43, 0x0020}, +{0x011F50, 0x0002}, +{0x011F5A, 0x0080}, +{0x011FB0, 0x0004}, +{0x011FB1, 0x0080}, +{0x011FC0, 0x0002}, +{0x011FD5, 0x0040}, +{0x011FF2, 0x0080}, +{0x011FFF, 0x0020}, +{0x012000, 0x0004}, +{0x01239A, 0x0080}, +{0x012400, 0x0002}, +{0x01246F, 0x0080}, +{0x012470, 0x0020}, +{0x012475, 0x0080}, +{0x012480, 0x0004}, +{0x012544, 0x0080}, +{0x012F90, 0x0004}, +{0x012FF1, 0x0020}, +{0x012FF3, 0x0080}, +{0x013000, 0x0004}, +{0x013430, 0x0080}, +{0x013440, 0x0010}, +{0x013441, 0x0004}, +{0x013447, 0x0010}, +{0x013456, 0x0080}, +{0x014400, 0x0004}, +{0x014647, 0x0080}, +{0x016800, 0x0004}, +{0x016A39, 0x0080}, +{0x016A40, 0x0004}, +{0x016A5F, 0x0080}, +{0x016A60, 0x0002}, +{0x016A6A, 0x0080}, +{0x016A6E, 0x0020}, +{0x016A70, 0x0004}, +{0x016ABF, 0x0080}, +{0x016AC0, 0x0002}, +{0x016ACA, 0x0080}, +{0x016AD0, 0x0004}, +{0x016AEE, 0x0080}, +{0x016AF0, 0x0010}, +{0x016AF5, 0x0020}, +{0x016AF6, 0x0080}, +{0x016B00, 0x0004}, +{0x016B30, 0x0010}, +{0x016B37, 0x0020}, +{0x016B3C, 0x0040}, +{0x016B40, 0x0004}, +{0x016B44, 0x0020}, +{0x016B45, 0x0040}, +{0x016B46, 0x0080}, +{0x016B50, 0x0002}, +{0x016B5A, 0x0080}, +{0x016B5B, 0x0002}, +{0x016B62, 0x0080}, +{0x016B63, 0x0004}, +{0x016B78, 0x0080}, +{0x016B7D, 0x0004}, +{0x016B90, 0x0080}, +{0x016E40, 0x0004}, +{0x016E80, 0x0002}, +{0x016E97, 0x0020}, +{0x016E9B, 0x0080}, +{0x016F00, 0x0004}, +{0x016F4B, 0x0080}, +{0x016F4F, 0x0010}, +{0x016F50, 0x0004}, +{0x016F51, 0x0010}, +{0x016F88, 0x0080}, +{0x016F8F, 0x0010}, +{0x016F93, 0x0004}, +{0x016FA0, 0x0080}, +{0x016FE0, 0x0004}, +{0x016FE2, 0x0020}, +{0x016FE3, 0x0004}, +{0x016FE4, 0x0010}, +{0x016FE5, 0x0080}, +{0x016FF0, 0x0010}, +{0x016FF2, 0x0080}, +{0x017000, 0x0004}, +{0x0187F8, 0x0080}, +{0x018800, 0x0004}, +{0x018CD6, 0x0080}, +{0x018D00, 0x0004}, +{0x018D09, 0x0080}, +{0x01AFF0, 0x0004}, +{0x01AFF4, 0x0080}, +{0x01AFF5, 0x0004}, +{0x01AFFC, 0x0080}, +{0x01AFFD, 0x0004}, +{0x01AFFF, 0x0080}, +{0x01B000, 0x0004}, +{0x01B123, 0x0080}, +{0x01B132, 0x0004}, +{0x01B133, 0x0080}, +{0x01B150, 0x0004}, +{0x01B153, 0x0080}, +{0x01B155, 0x0004}, +{0x01B156, 0x0080}, +{0x01B164, 0x0004}, +{0x01B168, 0x0080}, +{0x01B170, 0x0004}, +{0x01B2FC, 0x0080}, +{0x01BC00, 0x0004}, +{0x01BC6B, 0x0080}, +{0x01BC70, 0x0004}, +{0x01BC7D, 0x0080}, +{0x01BC80, 0x0004}, +{0x01BC89, 0x0080}, +{0x01BC90, 0x0004}, +{0x01BC9A, 0x0080}, +{0x01BC9C, 0x0040}, +{0x01BC9D, 0x0010}, +{0x01BC9F, 0x0020}, +{0x01BCA0, 0x0080}, +{0x01CF00, 0x0010}, +{0x01CF2E, 0x0080}, +{0x01CF30, 0x0010}, +{0x01CF47, 0x0080}, +{0x01CF50, 0x0040}, +{0x01CFC4, 0x0080}, +{0x01D000, 0x0040}, +{0x01D0F6, 0x0080}, +{0x01D100, 0x0040}, +{0x01D127, 0x0080}, +{0x01D129, 0x0040}, +{0x01D165, 0x0010}, +{0x01D16A, 0x0040}, +{0x01D16D, 0x0010}, +{0x01D173, 0x0080}, +{0x01D17B, 0x0010}, +{0x01D183, 0x0040}, +{0x01D185, 0x0010}, +{0x01D18C, 0x0040}, +{0x01D1AA, 0x0010}, +{0x01D1AE, 0x0040}, +{0x01D1EB, 0x0080}, +{0x01D200, 0x0040}, +{0x01D242, 0x0010}, +{0x01D245, 0x0040}, +{0x01D246, 0x0080}, +{0x01D2C0, 0x0002}, +{0x01D2D4, 0x0080}, +{0x01D2E0, 0x0002}, +{0x01D2F4, 0x0080}, +{0x01D300, 0x0040}, +{0x01D357, 0x0080}, +{0x01D360, 0x0002}, +{0x01D379, 0x0080}, +{0x01D400, 0x0004}, +{0x01D455, 0x0080}, +{0x01D456, 0x0004}, +{0x01D49D, 0x0080}, +{0x01D49E, 0x0004}, +{0x01D4A0, 0x0080}, +{0x01D4A2, 0x0004}, +{0x01D4A3, 0x0080}, +{0x01D4A5, 0x0004}, +{0x01D4A7, 0x0080}, +{0x01D4A9, 0x0004}, +{0x01D4AD, 0x0080}, +{0x01D4AE, 0x0004}, +{0x01D4BA, 0x0080}, +{0x01D4BB, 0x0004}, +{0x01D4BC, 0x0080}, +{0x01D4BD, 0x0004}, +{0x01D4C4, 0x0080}, +{0x01D4C5, 0x0004}, +{0x01D506, 0x0080}, +{0x01D507, 0x0004}, +{0x01D50B, 0x0080}, +{0x01D50D, 0x0004}, +{0x01D515, 0x0080}, +{0x01D516, 0x0004}, +{0x01D51D, 0x0080}, +{0x01D51E, 0x0004}, +{0x01D53A, 0x0080}, +{0x01D53B, 0x0004}, +{0x01D53F, 0x0080}, +{0x01D540, 0x0004}, +{0x01D545, 0x0080}, +{0x01D546, 0x0004}, +{0x01D547, 0x0080}, +{0x01D54A, 0x0004}, +{0x01D551, 0x0080}, +{0x01D552, 0x0004}, +{0x01D6A6, 0x0080}, +{0x01D6A8, 0x0004}, +{0x01D6C1, 0x0040}, +{0x01D6C2, 0x0004}, +{0x01D6DB, 0x0040}, +{0x01D6DC, 0x0004}, +{0x01D6FB, 0x0040}, +{0x01D6FC, 0x0004}, +{0x01D715, 0x0040}, +{0x01D716, 0x0004}, +{0x01D735, 0x0040}, +{0x01D736, 0x0004}, +{0x01D74F, 0x0040}, +{0x01D750, 0x0004}, +{0x01D76F, 0x0040}, +{0x01D770, 0x0004}, +{0x01D789, 0x0040}, +{0x01D78A, 0x0004}, +{0x01D7A9, 0x0040}, +{0x01D7AA, 0x0004}, +{0x01D7C3, 0x0040}, +{0x01D7C4, 0x0004}, +{0x01D7CC, 0x0080}, +{0x01D7CE, 0x0002}, +{0x01D800, 0x0040}, +{0x01DA00, 0x0010}, +{0x01DA37, 0x0040}, +{0x01DA3B, 0x0010}, +{0x01DA6D, 0x0040}, +{0x01DA75, 0x0010}, +{0x01DA76, 0x0040}, +{0x01DA84, 0x0010}, +{0x01DA85, 0x0040}, +{0x01DA87, 0x0020}, +{0x01DA8C, 0x0080}, +{0x01DA9B, 0x0010}, +{0x01DAA0, 0x0080}, +{0x01DAA1, 0x0010}, +{0x01DAB0, 0x0080}, +{0x01DF00, 0x0004}, +{0x01DF1F, 0x0080}, +{0x01DF25, 0x0004}, +{0x01DF2B, 0x0080}, +{0x01E000, 0x0010}, +{0x01E007, 0x0080}, +{0x01E008, 0x0010}, +{0x01E019, 0x0080}, +{0x01E01B, 0x0010}, +{0x01E022, 0x0080}, +{0x01E023, 0x0010}, +{0x01E025, 0x0080}, +{0x01E026, 0x0010}, +{0x01E02B, 0x0080}, +{0x01E030, 0x0004}, +{0x01E06E, 0x0080}, +{0x01E08F, 0x0010}, +{0x01E090, 0x0080}, +{0x01E100, 0x0004}, +{0x01E12D, 0x0080}, +{0x01E130, 0x0010}, +{0x01E137, 0x0004}, +{0x01E13E, 0x0080}, +{0x01E140, 0x0002}, +{0x01E14A, 0x0080}, +{0x01E14E, 0x0004}, +{0x01E14F, 0x0040}, +{0x01E150, 0x0080}, +{0x01E290, 0x0004}, +{0x01E2AE, 0x0010}, +{0x01E2AF, 0x0080}, +{0x01E2C0, 0x0004}, +{0x01E2EC, 0x0010}, +{0x01E2F0, 0x0002}, +{0x01E2FA, 0x0080}, +{0x01E2FF, 0x0040}, +{0x01E300, 0x0080}, +{0x01E4D0, 0x0004}, +{0x01E4EC, 0x0010}, +{0x01E4F0, 0x0002}, +{0x01E4FA, 0x0080}, +{0x01E7E0, 0x0004}, +{0x01E7E7, 0x0080}, +{0x01E7E8, 0x0004}, +{0x01E7EC, 0x0080}, +{0x01E7ED, 0x0004}, +{0x01E7EF, 0x0080}, +{0x01E7F0, 0x0004}, +{0x01E7FF, 0x0080}, +{0x01E800, 0x0004}, +{0x01E8C5, 0x0080}, +{0x01E8C7, 0x0002}, +{0x01E8D0, 0x0010}, +{0x01E8D7, 0x0080}, +{0x01E900, 0x0004}, +{0x01E944, 0x0010}, +{0x01E94B, 0x0004}, +{0x01E94C, 0x0080}, +{0x01E950, 0x0002}, +{0x01E95A, 0x0080}, +{0x01E95E, 0x0020}, +{0x01E960, 0x0080}, +{0x01EC71, 0x0002}, +{0x01ECAC, 0x0040}, +{0x01ECAD, 0x0002}, +{0x01ECB0, 0x0040}, +{0x01ECB1, 0x0002}, +{0x01ECB5, 0x0080}, +{0x01ED01, 0x0002}, +{0x01ED2E, 0x0040}, +{0x01ED2F, 0x0002}, +{0x01ED3E, 0x0080}, +{0x01EE00, 0x0004}, +{0x01EE04, 0x0080}, +{0x01EE05, 0x0004}, +{0x01EE20, 0x0080}, +{0x01EE21, 0x0004}, +{0x01EE23, 0x0080}, +{0x01EE24, 0x0004}, +{0x01EE25, 0x0080}, +{0x01EE27, 0x0004}, +{0x01EE28, 0x0080}, +{0x01EE29, 0x0004}, +{0x01EE33, 0x0080}, +{0x01EE34, 0x0004}, +{0x01EE38, 0x0080}, +{0x01EE39, 0x0004}, +{0x01EE3A, 0x0080}, +{0x01EE3B, 0x0004}, +{0x01EE3C, 0x0080}, +{0x01EE42, 0x0004}, +{0x01EE43, 0x0080}, +{0x01EE47, 0x0004}, +{0x01EE48, 0x0080}, +{0x01EE49, 0x0004}, +{0x01EE4A, 0x0080}, +{0x01EE4B, 0x0004}, +{0x01EE4C, 0x0080}, +{0x01EE4D, 0x0004}, +{0x01EE50, 0x0080}, +{0x01EE51, 0x0004}, +{0x01EE53, 0x0080}, +{0x01EE54, 0x0004}, +{0x01EE55, 0x0080}, +{0x01EE57, 0x0004}, +{0x01EE58, 0x0080}, +{0x01EE59, 0x0004}, +{0x01EE5A, 0x0080}, +{0x01EE5B, 0x0004}, +{0x01EE5C, 0x0080}, +{0x01EE5D, 0x0004}, +{0x01EE5E, 0x0080}, +{0x01EE5F, 0x0004}, +{0x01EE60, 0x0080}, +{0x01EE61, 0x0004}, +{0x01EE63, 0x0080}, +{0x01EE64, 0x0004}, +{0x01EE65, 0x0080}, +{0x01EE67, 0x0004}, +{0x01EE6B, 0x0080}, +{0x01EE6C, 0x0004}, +{0x01EE73, 0x0080}, +{0x01EE74, 0x0004}, +{0x01EE78, 0x0080}, +{0x01EE79, 0x0004}, +{0x01EE7D, 0x0080}, +{0x01EE7E, 0x0004}, +{0x01EE7F, 0x0080}, +{0x01EE80, 0x0004}, +{0x01EE8A, 0x0080}, +{0x01EE8B, 0x0004}, +{0x01EE9C, 0x0080}, +{0x01EEA1, 0x0004}, +{0x01EEA4, 0x0080}, +{0x01EEA5, 0x0004}, +{0x01EEAA, 0x0080}, +{0x01EEAB, 0x0004}, +{0x01EEBC, 0x0080}, +{0x01EEF0, 0x0040}, +{0x01EEF2, 0x0080}, +{0x01F000, 0x0040}, +{0x01F02C, 0x0080}, +{0x01F030, 0x0040}, +{0x01F094, 0x0080}, +{0x01F0A0, 0x0040}, +{0x01F0AF, 0x0080}, +{0x01F0B1, 0x0040}, +{0x01F0C0, 0x0080}, +{0x01F0C1, 0x0040}, +{0x01F0D0, 0x0080}, +{0x01F0D1, 0x0040}, +{0x01F0F6, 0x0080}, +{0x01F100, 0x0002}, +{0x01F10D, 0x0040}, +{0x01F1AE, 0x0080}, +{0x01F1E6, 0x0040}, +{0x01F203, 0x0080}, +{0x01F210, 0x0040}, +{0x01F23C, 0x0080}, +{0x01F240, 0x0040}, +{0x01F249, 0x0080}, +{0x01F250, 0x0040}, +{0x01F252, 0x0080}, +{0x01F260, 0x0040}, +{0x01F266, 0x0080}, +{0x01F300, 0x0040}, +{0x01F6D8, 0x0080}, +{0x01F6DC, 0x0040}, +{0x01F6ED, 0x0080}, +{0x01F6F0, 0x0040}, +{0x01F6FD, 0x0080}, +{0x01F700, 0x0040}, +{0x01F777, 0x0080}, +{0x01F77B, 0x0040}, +{0x01F7DA, 0x0080}, +{0x01F7E0, 0x0040}, +{0x01F7EC, 0x0080}, +{0x01F7F0, 0x0040}, +{0x01F7F1, 0x0080}, +{0x01F800, 0x0040}, +{0x01F80C, 0x0080}, +{0x01F810, 0x0040}, +{0x01F848, 0x0080}, +{0x01F850, 0x0040}, +{0x01F85A, 0x0080}, +{0x01F860, 0x0040}, +{0x01F888, 0x0080}, +{0x01F890, 0x0040}, +{0x01F8AE, 0x0080}, +{0x01F8B0, 0x0040}, +{0x01F8B2, 0x0080}, +{0x01F900, 0x0040}, +{0x01FA54, 0x0080}, +{0x01FA60, 0x0040}, +{0x01FA6E, 0x0080}, +{0x01FA70, 0x0040}, +{0x01FA7D, 0x0080}, +{0x01FA80, 0x0040}, +{0x01FA89, 0x0080}, +{0x01FA90, 0x0040}, +{0x01FABE, 0x0080}, +{0x01FABF, 0x0040}, +{0x01FAC6, 0x0080}, +{0x01FACE, 0x0040}, +{0x01FADC, 0x0080}, +{0x01FAE0, 0x0040}, +{0x01FAE9, 0x0080}, +{0x01FAF0, 0x0040}, +{0x01FAF9, 0x0080}, +{0x01FB00, 0x0040}, +{0x01FB93, 0x0080}, +{0x01FB94, 0x0040}, +{0x01FBCB, 0x0080}, +{0x01FBF0, 0x0002}, +{0x01FBFA, 0x0080}, +{0x020000, 0x0004}, +{0x02A6E0, 0x0080}, +{0x02A700, 0x0004}, +{0x02B73A, 0x0080}, +{0x02B740, 0x0004}, +{0x02B81E, 0x0080}, +{0x02B820, 0x0004}, +{0x02CEA2, 0x0080}, +{0x02CEB0, 0x0004}, +{0x02EBE1, 0x0080}, +{0x02EBF0, 0x0004}, +{0x02EE5E, 0x0080}, +{0x02F800, 0x0004}, +{0x02FA1E, 0x0080}, +{0x030000, 0x0004}, +{0x03134B, 0x0080}, +{0x031350, 0x0004}, +{0x0323B0, 0x0080}, +{0x0E0100, 0x0010}, +{0x0E01F0, 0x0080}, +{0x110000, 0x0000}, }; -const std::vector> unicode_ranges_letter = { -{0x00000041, 0x0000005A}, {0x00000061, 0x0000007A}, {0x000000AA, 0x000000AA}, {0x000000B5, 0x000000B5}, -{0x000000BA, 0x000000BA}, {0x000000C0, 0x000000D6}, {0x000000D8, 0x000000F6}, {0x000000F8, 0x000002C1}, -{0x000002C6, 0x000002D1}, {0x000002E0, 0x000002E4}, {0x000002EC, 0x000002EC}, {0x000002EE, 0x000002EE}, -{0x00000370, 0x00000374}, {0x00000376, 0x00000377}, {0x0000037A, 0x0000037D}, {0x0000037F, 0x0000037F}, -{0x00000386, 0x00000386}, {0x00000388, 0x0000038A}, {0x0000038C, 0x0000038C}, {0x0000038E, 0x000003A1}, -{0x000003A3, 0x000003F5}, {0x000003F7, 0x00000481}, {0x0000048A, 0x0000052F}, {0x00000531, 0x00000556}, -{0x00000559, 0x00000559}, {0x00000560, 0x00000588}, {0x000005D0, 0x000005EA}, {0x000005EF, 0x000005F2}, -{0x00000620, 0x0000064A}, {0x0000066E, 0x0000066F}, {0x00000671, 0x000006D3}, {0x000006D5, 0x000006D5}, -{0x000006E5, 0x000006E6}, {0x000006EE, 0x000006EF}, {0x000006FA, 0x000006FC}, {0x000006FF, 0x000006FF}, -{0x00000710, 0x00000710}, {0x00000712, 0x0000072F}, {0x0000074D, 0x000007A5}, {0x000007B1, 0x000007B1}, -{0x000007CA, 0x000007EA}, {0x000007F4, 0x000007F5}, {0x000007FA, 0x000007FA}, {0x00000800, 0x00000815}, -{0x0000081A, 0x0000081A}, {0x00000824, 0x00000824}, {0x00000828, 0x00000828}, {0x00000840, 0x00000858}, -{0x00000860, 0x0000086A}, {0x00000870, 0x00000887}, {0x00000889, 0x0000088E}, {0x000008A0, 0x000008C9}, -{0x00000904, 0x00000939}, {0x0000093D, 0x0000093D}, {0x00000950, 0x00000950}, {0x00000958, 0x00000961}, -{0x00000971, 0x00000980}, {0x00000985, 0x0000098C}, {0x0000098F, 0x00000990}, {0x00000993, 0x000009A8}, -{0x000009AA, 0x000009B0}, {0x000009B2, 0x000009B2}, {0x000009B6, 0x000009B9}, {0x000009BD, 0x000009BD}, -{0x000009CE, 0x000009CE}, {0x000009DC, 0x000009DD}, {0x000009DF, 0x000009E1}, {0x000009F0, 0x000009F1}, -{0x000009FC, 0x000009FC}, {0x00000A05, 0x00000A0A}, {0x00000A0F, 0x00000A10}, {0x00000A13, 0x00000A28}, -{0x00000A2A, 0x00000A30}, {0x00000A32, 0x00000A33}, {0x00000A35, 0x00000A36}, {0x00000A38, 0x00000A39}, -{0x00000A59, 0x00000A5C}, {0x00000A5E, 0x00000A5E}, {0x00000A72, 0x00000A74}, {0x00000A85, 0x00000A8D}, -{0x00000A8F, 0x00000A91}, {0x00000A93, 0x00000AA8}, {0x00000AAA, 0x00000AB0}, {0x00000AB2, 0x00000AB3}, -{0x00000AB5, 0x00000AB9}, {0x00000ABD, 0x00000ABD}, {0x00000AD0, 0x00000AD0}, {0x00000AE0, 0x00000AE1}, -{0x00000AF9, 0x00000AF9}, {0x00000B05, 0x00000B0C}, {0x00000B0F, 0x00000B10}, {0x00000B13, 0x00000B28}, -{0x00000B2A, 0x00000B30}, {0x00000B32, 0x00000B33}, {0x00000B35, 0x00000B39}, {0x00000B3D, 0x00000B3D}, -{0x00000B5C, 0x00000B5D}, {0x00000B5F, 0x00000B61}, {0x00000B71, 0x00000B71}, {0x00000B83, 0x00000B83}, -{0x00000B85, 0x00000B8A}, {0x00000B8E, 0x00000B90}, {0x00000B92, 0x00000B95}, {0x00000B99, 0x00000B9A}, -{0x00000B9C, 0x00000B9C}, {0x00000B9E, 0x00000B9F}, {0x00000BA3, 0x00000BA4}, {0x00000BA8, 0x00000BAA}, -{0x00000BAE, 0x00000BB9}, {0x00000BD0, 0x00000BD0}, {0x00000C05, 0x00000C0C}, {0x00000C0E, 0x00000C10}, -{0x00000C12, 0x00000C28}, {0x00000C2A, 0x00000C39}, {0x00000C3D, 0x00000C3D}, {0x00000C58, 0x00000C5A}, -{0x00000C5D, 0x00000C5D}, {0x00000C60, 0x00000C61}, {0x00000C80, 0x00000C80}, {0x00000C85, 0x00000C8C}, -{0x00000C8E, 0x00000C90}, {0x00000C92, 0x00000CA8}, {0x00000CAA, 0x00000CB3}, {0x00000CB5, 0x00000CB9}, -{0x00000CBD, 0x00000CBD}, {0x00000CDD, 0x00000CDE}, {0x00000CE0, 0x00000CE1}, {0x00000CF1, 0x00000CF2}, -{0x00000D04, 0x00000D0C}, {0x00000D0E, 0x00000D10}, {0x00000D12, 0x00000D3A}, {0x00000D3D, 0x00000D3D}, -{0x00000D4E, 0x00000D4E}, {0x00000D54, 0x00000D56}, {0x00000D5F, 0x00000D61}, {0x00000D7A, 0x00000D7F}, -{0x00000D85, 0x00000D96}, {0x00000D9A, 0x00000DB1}, {0x00000DB3, 0x00000DBB}, {0x00000DBD, 0x00000DBD}, -{0x00000DC0, 0x00000DC6}, {0x00000E01, 0x00000E30}, {0x00000E32, 0x00000E33}, {0x00000E40, 0x00000E46}, -{0x00000E81, 0x00000E82}, {0x00000E84, 0x00000E84}, {0x00000E86, 0x00000E8A}, {0x00000E8C, 0x00000EA3}, -{0x00000EA5, 0x00000EA5}, {0x00000EA7, 0x00000EB0}, {0x00000EB2, 0x00000EB3}, {0x00000EBD, 0x00000EBD}, -{0x00000EC0, 0x00000EC4}, {0x00000EC6, 0x00000EC6}, {0x00000EDC, 0x00000EDF}, {0x00000F00, 0x00000F00}, -{0x00000F40, 0x00000F47}, {0x00000F49, 0x00000F6C}, {0x00000F88, 0x00000F8C}, {0x00001000, 0x0000102A}, -{0x0000103F, 0x0000103F}, {0x00001050, 0x00001055}, {0x0000105A, 0x0000105D}, {0x00001061, 0x00001061}, -{0x00001065, 0x00001066}, {0x0000106E, 0x00001070}, {0x00001075, 0x00001081}, {0x0000108E, 0x0000108E}, -{0x000010A0, 0x000010C5}, {0x000010C7, 0x000010C7}, {0x000010CD, 0x000010CD}, {0x000010D0, 0x000010FA}, -{0x000010FC, 0x00001248}, {0x0000124A, 0x0000124D}, {0x00001250, 0x00001256}, {0x00001258, 0x00001258}, -{0x0000125A, 0x0000125D}, {0x00001260, 0x00001288}, {0x0000128A, 0x0000128D}, {0x00001290, 0x000012B0}, -{0x000012B2, 0x000012B5}, {0x000012B8, 0x000012BE}, {0x000012C0, 0x000012C0}, {0x000012C2, 0x000012C5}, -{0x000012C8, 0x000012D6}, {0x000012D8, 0x00001310}, {0x00001312, 0x00001315}, {0x00001318, 0x0000135A}, -{0x00001380, 0x0000138F}, {0x000013A0, 0x000013F5}, {0x000013F8, 0x000013FD}, {0x00001401, 0x0000166C}, -{0x0000166F, 0x0000167F}, {0x00001681, 0x0000169A}, {0x000016A0, 0x000016EA}, {0x000016F1, 0x000016F8}, -{0x00001700, 0x00001711}, {0x0000171F, 0x00001731}, {0x00001740, 0x00001751}, {0x00001760, 0x0000176C}, -{0x0000176E, 0x00001770}, {0x00001780, 0x000017B3}, {0x000017D7, 0x000017D7}, {0x000017DC, 0x000017DC}, -{0x00001820, 0x00001878}, {0x00001880, 0x00001884}, {0x00001887, 0x000018A8}, {0x000018AA, 0x000018AA}, -{0x000018B0, 0x000018F5}, {0x00001900, 0x0000191E}, {0x00001950, 0x0000196D}, {0x00001970, 0x00001974}, -{0x00001980, 0x000019AB}, {0x000019B0, 0x000019C9}, {0x00001A00, 0x00001A16}, {0x00001A20, 0x00001A54}, -{0x00001AA7, 0x00001AA7}, {0x00001B05, 0x00001B33}, {0x00001B45, 0x00001B4C}, {0x00001B83, 0x00001BA0}, -{0x00001BAE, 0x00001BAF}, {0x00001BBA, 0x00001BE5}, {0x00001C00, 0x00001C23}, {0x00001C4D, 0x00001C4F}, -{0x00001C5A, 0x00001C7D}, {0x00001C80, 0x00001C88}, {0x00001C90, 0x00001CBA}, {0x00001CBD, 0x00001CBF}, -{0x00001CE9, 0x00001CEC}, {0x00001CEE, 0x00001CF3}, {0x00001CF5, 0x00001CF6}, {0x00001CFA, 0x00001CFA}, -{0x00001D00, 0x00001DBF}, {0x00001E00, 0x00001F15}, {0x00001F18, 0x00001F1D}, {0x00001F20, 0x00001F45}, -{0x00001F48, 0x00001F4D}, {0x00001F50, 0x00001F57}, {0x00001F59, 0x00001F59}, {0x00001F5B, 0x00001F5B}, -{0x00001F5D, 0x00001F5D}, {0x00001F5F, 0x00001F7D}, {0x00001F80, 0x00001FB4}, {0x00001FB6, 0x00001FBC}, -{0x00001FBE, 0x00001FBE}, {0x00001FC2, 0x00001FC4}, {0x00001FC6, 0x00001FCC}, {0x00001FD0, 0x00001FD3}, -{0x00001FD6, 0x00001FDB}, {0x00001FE0, 0x00001FEC}, {0x00001FF2, 0x00001FF4}, {0x00001FF6, 0x00001FFC}, -{0x00002071, 0x00002071}, {0x0000207F, 0x0000207F}, {0x00002090, 0x0000209C}, {0x00002102, 0x00002102}, -{0x00002107, 0x00002107}, {0x0000210A, 0x00002113}, {0x00002115, 0x00002115}, {0x00002119, 0x0000211D}, -{0x00002124, 0x00002124}, {0x00002126, 0x00002126}, {0x00002128, 0x00002128}, {0x0000212A, 0x0000212D}, -{0x0000212F, 0x00002139}, {0x0000213C, 0x0000213F}, {0x00002145, 0x00002149}, {0x0000214E, 0x0000214E}, -{0x00002183, 0x00002184}, {0x00002C00, 0x00002CE4}, {0x00002CEB, 0x00002CEE}, {0x00002CF2, 0x00002CF3}, -{0x00002D00, 0x00002D25}, {0x00002D27, 0x00002D27}, {0x00002D2D, 0x00002D2D}, {0x00002D30, 0x00002D67}, -{0x00002D6F, 0x00002D6F}, {0x00002D80, 0x00002D96}, {0x00002DA0, 0x00002DA6}, {0x00002DA8, 0x00002DAE}, -{0x00002DB0, 0x00002DB6}, {0x00002DB8, 0x00002DBE}, {0x00002DC0, 0x00002DC6}, {0x00002DC8, 0x00002DCE}, -{0x00002DD0, 0x00002DD6}, {0x00002DD8, 0x00002DDE}, {0x00002E2F, 0x00002E2F}, {0x00003005, 0x00003006}, -{0x00003031, 0x00003035}, {0x0000303B, 0x0000303C}, {0x00003041, 0x00003096}, {0x0000309D, 0x0000309F}, -{0x000030A1, 0x000030FA}, {0x000030FC, 0x000030FF}, {0x00003105, 0x0000312F}, {0x00003131, 0x0000318E}, -{0x000031A0, 0x000031BF}, {0x000031F0, 0x000031FF}, {0x00003400, 0x00004DBF}, {0x00004E00, 0x0000A48C}, -{0x0000A4D0, 0x0000A4FD}, {0x0000A500, 0x0000A60C}, {0x0000A610, 0x0000A61F}, {0x0000A62A, 0x0000A62B}, -{0x0000A640, 0x0000A66E}, {0x0000A67F, 0x0000A69D}, {0x0000A6A0, 0x0000A6E5}, {0x0000A717, 0x0000A71F}, -{0x0000A722, 0x0000A788}, {0x0000A78B, 0x0000A7CA}, {0x0000A7D0, 0x0000A7D1}, {0x0000A7D3, 0x0000A7D3}, -{0x0000A7D5, 0x0000A7D9}, {0x0000A7F2, 0x0000A801}, {0x0000A803, 0x0000A805}, {0x0000A807, 0x0000A80A}, -{0x0000A80C, 0x0000A822}, {0x0000A840, 0x0000A873}, {0x0000A882, 0x0000A8B3}, {0x0000A8F2, 0x0000A8F7}, -{0x0000A8FB, 0x0000A8FB}, {0x0000A8FD, 0x0000A8FE}, {0x0000A90A, 0x0000A925}, {0x0000A930, 0x0000A946}, -{0x0000A960, 0x0000A97C}, {0x0000A984, 0x0000A9B2}, {0x0000A9CF, 0x0000A9CF}, {0x0000A9E0, 0x0000A9E4}, -{0x0000A9E6, 0x0000A9EF}, {0x0000A9FA, 0x0000A9FE}, {0x0000AA00, 0x0000AA28}, {0x0000AA40, 0x0000AA42}, -{0x0000AA44, 0x0000AA4B}, {0x0000AA60, 0x0000AA76}, {0x0000AA7A, 0x0000AA7A}, {0x0000AA7E, 0x0000AAAF}, -{0x0000AAB1, 0x0000AAB1}, {0x0000AAB5, 0x0000AAB6}, {0x0000AAB9, 0x0000AABD}, {0x0000AAC0, 0x0000AAC0}, -{0x0000AAC2, 0x0000AAC2}, {0x0000AADB, 0x0000AADD}, {0x0000AAE0, 0x0000AAEA}, {0x0000AAF2, 0x0000AAF4}, -{0x0000AB01, 0x0000AB06}, {0x0000AB09, 0x0000AB0E}, {0x0000AB11, 0x0000AB16}, {0x0000AB20, 0x0000AB26}, -{0x0000AB28, 0x0000AB2E}, {0x0000AB30, 0x0000AB5A}, {0x0000AB5C, 0x0000AB69}, {0x0000AB70, 0x0000ABE2}, -{0x0000AC00, 0x0000D7A3}, {0x0000D7B0, 0x0000D7C6}, {0x0000D7CB, 0x0000D7FB}, {0x0000F900, 0x0000FA6D}, -{0x0000FA70, 0x0000FAD9}, {0x0000FB00, 0x0000FB06}, {0x0000FB13, 0x0000FB17}, {0x0000FB1D, 0x0000FB1D}, -{0x0000FB1F, 0x0000FB28}, {0x0000FB2A, 0x0000FB36}, {0x0000FB38, 0x0000FB3C}, {0x0000FB3E, 0x0000FB3E}, -{0x0000FB40, 0x0000FB41}, {0x0000FB43, 0x0000FB44}, {0x0000FB46, 0x0000FBB1}, {0x0000FBD3, 0x0000FD3D}, -{0x0000FD50, 0x0000FD8F}, {0x0000FD92, 0x0000FDC7}, {0x0000FDF0, 0x0000FDFB}, {0x0000FE70, 0x0000FE74}, -{0x0000FE76, 0x0000FEFC}, {0x0000FF21, 0x0000FF3A}, {0x0000FF41, 0x0000FF5A}, {0x0000FF66, 0x0000FFBE}, -{0x0000FFC2, 0x0000FFC7}, {0x0000FFCA, 0x0000FFCF}, {0x0000FFD2, 0x0000FFD7}, {0x0000FFDA, 0x0000FFDC}, -{0x00010000, 0x0001000B}, {0x0001000D, 0x00010026}, {0x00010028, 0x0001003A}, {0x0001003C, 0x0001003D}, -{0x0001003F, 0x0001004D}, {0x00010050, 0x0001005D}, {0x00010080, 0x000100FA}, {0x00010280, 0x0001029C}, -{0x000102A0, 0x000102D0}, {0x00010300, 0x0001031F}, {0x0001032D, 0x00010340}, {0x00010342, 0x00010349}, -{0x00010350, 0x00010375}, {0x00010380, 0x0001039D}, {0x000103A0, 0x000103C3}, {0x000103C8, 0x000103CF}, -{0x00010400, 0x0001049D}, {0x000104B0, 0x000104D3}, {0x000104D8, 0x000104FB}, {0x00010500, 0x00010527}, -{0x00010530, 0x00010563}, {0x00010570, 0x0001057A}, {0x0001057C, 0x0001058A}, {0x0001058C, 0x00010592}, -{0x00010594, 0x00010595}, {0x00010597, 0x000105A1}, {0x000105A3, 0x000105B1}, {0x000105B3, 0x000105B9}, -{0x000105BB, 0x000105BC}, {0x00010600, 0x00010736}, {0x00010740, 0x00010755}, {0x00010760, 0x00010767}, -{0x00010780, 0x00010785}, {0x00010787, 0x000107B0}, {0x000107B2, 0x000107BA}, {0x00010800, 0x00010805}, -{0x00010808, 0x00010808}, {0x0001080A, 0x00010835}, {0x00010837, 0x00010838}, {0x0001083C, 0x0001083C}, -{0x0001083F, 0x00010855}, {0x00010860, 0x00010876}, {0x00010880, 0x0001089E}, {0x000108E0, 0x000108F2}, -{0x000108F4, 0x000108F5}, {0x00010900, 0x00010915}, {0x00010920, 0x00010939}, {0x00010980, 0x000109B7}, -{0x000109BE, 0x000109BF}, {0x00010A00, 0x00010A00}, {0x00010A10, 0x00010A13}, {0x00010A15, 0x00010A17}, -{0x00010A19, 0x00010A35}, {0x00010A60, 0x00010A7C}, {0x00010A80, 0x00010A9C}, {0x00010AC0, 0x00010AC7}, -{0x00010AC9, 0x00010AE4}, {0x00010B00, 0x00010B35}, {0x00010B40, 0x00010B55}, {0x00010B60, 0x00010B72}, -{0x00010B80, 0x00010B91}, {0x00010C00, 0x00010C48}, {0x00010C80, 0x00010CB2}, {0x00010CC0, 0x00010CF2}, -{0x00010D00, 0x00010D23}, {0x00010E80, 0x00010EA9}, {0x00010EB0, 0x00010EB1}, {0x00010F00, 0x00010F1C}, -{0x00010F27, 0x00010F27}, {0x00010F30, 0x00010F45}, {0x00010F70, 0x00010F81}, {0x00010FB0, 0x00010FC4}, -{0x00010FE0, 0x00010FF6}, {0x00011003, 0x00011037}, {0x00011071, 0x00011072}, {0x00011075, 0x00011075}, -{0x00011083, 0x000110AF}, {0x000110D0, 0x000110E8}, {0x00011103, 0x00011126}, {0x00011144, 0x00011144}, -{0x00011147, 0x00011147}, {0x00011150, 0x00011172}, {0x00011176, 0x00011176}, {0x00011183, 0x000111B2}, -{0x000111C1, 0x000111C4}, {0x000111DA, 0x000111DA}, {0x000111DC, 0x000111DC}, {0x00011200, 0x00011211}, -{0x00011213, 0x0001122B}, {0x0001123F, 0x00011240}, {0x00011280, 0x00011286}, {0x00011288, 0x00011288}, -{0x0001128A, 0x0001128D}, {0x0001128F, 0x0001129D}, {0x0001129F, 0x000112A8}, {0x000112B0, 0x000112DE}, -{0x00011305, 0x0001130C}, {0x0001130F, 0x00011310}, {0x00011313, 0x00011328}, {0x0001132A, 0x00011330}, -{0x00011332, 0x00011333}, {0x00011335, 0x00011339}, {0x0001133D, 0x0001133D}, {0x00011350, 0x00011350}, -{0x0001135D, 0x00011361}, {0x00011400, 0x00011434}, {0x00011447, 0x0001144A}, {0x0001145F, 0x00011461}, -{0x00011480, 0x000114AF}, {0x000114C4, 0x000114C5}, {0x000114C7, 0x000114C7}, {0x00011580, 0x000115AE}, -{0x000115D8, 0x000115DB}, {0x00011600, 0x0001162F}, {0x00011644, 0x00011644}, {0x00011680, 0x000116AA}, -{0x000116B8, 0x000116B8}, {0x00011700, 0x0001171A}, {0x00011740, 0x00011746}, {0x00011800, 0x0001182B}, -{0x000118A0, 0x000118DF}, {0x000118FF, 0x00011906}, {0x00011909, 0x00011909}, {0x0001190C, 0x00011913}, -{0x00011915, 0x00011916}, {0x00011918, 0x0001192F}, {0x0001193F, 0x0001193F}, {0x00011941, 0x00011941}, -{0x000119A0, 0x000119A7}, {0x000119AA, 0x000119D0}, {0x000119E1, 0x000119E1}, {0x000119E3, 0x000119E3}, -{0x00011A00, 0x00011A00}, {0x00011A0B, 0x00011A32}, {0x00011A3A, 0x00011A3A}, {0x00011A50, 0x00011A50}, -{0x00011A5C, 0x00011A89}, {0x00011A9D, 0x00011A9D}, {0x00011AB0, 0x00011AF8}, {0x00011C00, 0x00011C08}, -{0x00011C0A, 0x00011C2E}, {0x00011C40, 0x00011C40}, {0x00011C72, 0x00011C8F}, {0x00011D00, 0x00011D06}, -{0x00011D08, 0x00011D09}, {0x00011D0B, 0x00011D30}, {0x00011D46, 0x00011D46}, {0x00011D60, 0x00011D65}, -{0x00011D67, 0x00011D68}, {0x00011D6A, 0x00011D89}, {0x00011D98, 0x00011D98}, {0x00011EE0, 0x00011EF2}, -{0x00011F02, 0x00011F02}, {0x00011F04, 0x00011F10}, {0x00011F12, 0x00011F33}, {0x00011FB0, 0x00011FB0}, -{0x00012000, 0x00012399}, {0x00012480, 0x00012543}, {0x00012F90, 0x00012FF0}, {0x00013000, 0x0001342F}, -{0x00013441, 0x00013446}, {0x00014400, 0x00014646}, {0x00016800, 0x00016A38}, {0x00016A40, 0x00016A5E}, -{0x00016A70, 0x00016ABE}, {0x00016AD0, 0x00016AED}, {0x00016B00, 0x00016B2F}, {0x00016B40, 0x00016B43}, -{0x00016B63, 0x00016B77}, {0x00016B7D, 0x00016B8F}, {0x00016E40, 0x00016E7F}, {0x00016F00, 0x00016F4A}, -{0x00016F50, 0x00016F50}, {0x00016F93, 0x00016F9F}, {0x00016FE0, 0x00016FE1}, {0x00016FE3, 0x00016FE3}, -{0x00017000, 0x000187F7}, {0x00018800, 0x00018CD5}, {0x00018D00, 0x00018D08}, {0x0001AFF0, 0x0001AFF3}, -{0x0001AFF5, 0x0001AFFB}, {0x0001AFFD, 0x0001AFFE}, {0x0001B000, 0x0001B122}, {0x0001B132, 0x0001B132}, -{0x0001B150, 0x0001B152}, {0x0001B155, 0x0001B155}, {0x0001B164, 0x0001B167}, {0x0001B170, 0x0001B2FB}, -{0x0001BC00, 0x0001BC6A}, {0x0001BC70, 0x0001BC7C}, {0x0001BC80, 0x0001BC88}, {0x0001BC90, 0x0001BC99}, -{0x0001D400, 0x0001D454}, {0x0001D456, 0x0001D49C}, {0x0001D49E, 0x0001D49F}, {0x0001D4A2, 0x0001D4A2}, -{0x0001D4A5, 0x0001D4A6}, {0x0001D4A9, 0x0001D4AC}, {0x0001D4AE, 0x0001D4B9}, {0x0001D4BB, 0x0001D4BB}, -{0x0001D4BD, 0x0001D4C3}, {0x0001D4C5, 0x0001D505}, {0x0001D507, 0x0001D50A}, {0x0001D50D, 0x0001D514}, -{0x0001D516, 0x0001D51C}, {0x0001D51E, 0x0001D539}, {0x0001D53B, 0x0001D53E}, {0x0001D540, 0x0001D544}, -{0x0001D546, 0x0001D546}, {0x0001D54A, 0x0001D550}, {0x0001D552, 0x0001D6A5}, {0x0001D6A8, 0x0001D6C0}, -{0x0001D6C2, 0x0001D6DA}, {0x0001D6DC, 0x0001D6FA}, {0x0001D6FC, 0x0001D714}, {0x0001D716, 0x0001D734}, -{0x0001D736, 0x0001D74E}, {0x0001D750, 0x0001D76E}, {0x0001D770, 0x0001D788}, {0x0001D78A, 0x0001D7A8}, -{0x0001D7AA, 0x0001D7C2}, {0x0001D7C4, 0x0001D7CB}, {0x0001DF00, 0x0001DF1E}, {0x0001DF25, 0x0001DF2A}, -{0x0001E030, 0x0001E06D}, {0x0001E100, 0x0001E12C}, {0x0001E137, 0x0001E13D}, {0x0001E14E, 0x0001E14E}, -{0x0001E290, 0x0001E2AD}, {0x0001E2C0, 0x0001E2EB}, {0x0001E4D0, 0x0001E4EB}, {0x0001E7E0, 0x0001E7E6}, -{0x0001E7E8, 0x0001E7EB}, {0x0001E7ED, 0x0001E7EE}, {0x0001E7F0, 0x0001E7FE}, {0x0001E800, 0x0001E8C4}, -{0x0001E900, 0x0001E943}, {0x0001E94B, 0x0001E94B}, {0x0001EE00, 0x0001EE03}, {0x0001EE05, 0x0001EE1F}, -{0x0001EE21, 0x0001EE22}, {0x0001EE24, 0x0001EE24}, {0x0001EE27, 0x0001EE27}, {0x0001EE29, 0x0001EE32}, -{0x0001EE34, 0x0001EE37}, {0x0001EE39, 0x0001EE39}, {0x0001EE3B, 0x0001EE3B}, {0x0001EE42, 0x0001EE42}, -{0x0001EE47, 0x0001EE47}, {0x0001EE49, 0x0001EE49}, {0x0001EE4B, 0x0001EE4B}, {0x0001EE4D, 0x0001EE4F}, -{0x0001EE51, 0x0001EE52}, {0x0001EE54, 0x0001EE54}, {0x0001EE57, 0x0001EE57}, {0x0001EE59, 0x0001EE59}, -{0x0001EE5B, 0x0001EE5B}, {0x0001EE5D, 0x0001EE5D}, {0x0001EE5F, 0x0001EE5F}, {0x0001EE61, 0x0001EE62}, -{0x0001EE64, 0x0001EE64}, {0x0001EE67, 0x0001EE6A}, {0x0001EE6C, 0x0001EE72}, {0x0001EE74, 0x0001EE77}, -{0x0001EE79, 0x0001EE7C}, {0x0001EE7E, 0x0001EE7E}, {0x0001EE80, 0x0001EE89}, {0x0001EE8B, 0x0001EE9B}, -{0x0001EEA1, 0x0001EEA3}, {0x0001EEA5, 0x0001EEA9}, {0x0001EEAB, 0x0001EEBB}, {0x00020000, 0x0002A6DF}, -{0x0002A700, 0x0002B739}, {0x0002B740, 0x0002B81D}, {0x0002B820, 0x0002CEA1}, {0x0002CEB0, 0x0002EBE0}, -{0x0002EBF0, 0x0002EE5D}, {0x0002F800, 0x0002FA1D}, {0x00030000, 0x0003134A}, {0x00031350, 0x000323AF}, +const std::unordered_set unicode_set_whitespace = { +0x000009, 0x00000A, 0x00000B, 0x00000C, 0x00000D, 0x000020, 0x000085, 0x0000A0, 0x001680, 0x002000, 0x002001, 0x002002, 0x002003, 0x002004, 0x002005, 0x002006, 0x002007, 0x002008, 0x002009, 0x00200A, 0x002028, 0x002029, 0x00202F, 0x00205F, 0x003000 }; -const std::vector> unicode_ranges_separator = { -{0x00000020, 0x00000020}, {0x000000A0, 0x000000A0}, {0x00001680, 0x00001680}, {0x00002000, 0x0000200A}, -{0x00002028, 0x00002029}, {0x0000202F, 0x0000202F}, {0x0000205F, 0x0000205F}, {0x00003000, 0x00003000}, +const std::unordered_map unicode_map_lowercase = { +{0x000041, 0x000061}, +{0x000042, 0x000062}, +{0x000043, 0x000063}, +{0x000044, 0x000064}, +{0x000045, 0x000065}, +{0x000046, 0x000066}, +{0x000047, 0x000067}, +{0x000048, 0x000068}, +{0x000049, 0x000069}, +{0x00004A, 0x00006A}, +{0x00004B, 0x00006B}, +{0x00004C, 0x00006C}, +{0x00004D, 0x00006D}, +{0x00004E, 0x00006E}, +{0x00004F, 0x00006F}, +{0x000050, 0x000070}, +{0x000051, 0x000071}, +{0x000052, 0x000072}, +{0x000053, 0x000073}, +{0x000054, 0x000074}, +{0x000055, 0x000075}, +{0x000056, 0x000076}, +{0x000057, 0x000077}, +{0x000058, 0x000078}, +{0x000059, 0x000079}, +{0x00005A, 0x00007A}, +{0x0000C0, 0x0000E0}, +{0x0000C1, 0x0000E1}, +{0x0000C2, 0x0000E2}, +{0x0000C3, 0x0000E3}, +{0x0000C4, 0x0000E4}, +{0x0000C5, 0x0000E5}, +{0x0000C6, 0x0000E6}, +{0x0000C7, 0x0000E7}, +{0x0000C8, 0x0000E8}, +{0x0000C9, 0x0000E9}, +{0x0000CA, 0x0000EA}, +{0x0000CB, 0x0000EB}, +{0x0000CC, 0x0000EC}, +{0x0000CD, 0x0000ED}, +{0x0000CE, 0x0000EE}, +{0x0000CF, 0x0000EF}, +{0x0000D0, 0x0000F0}, +{0x0000D1, 0x0000F1}, +{0x0000D2, 0x0000F2}, +{0x0000D3, 0x0000F3}, +{0x0000D4, 0x0000F4}, +{0x0000D5, 0x0000F5}, +{0x0000D6, 0x0000F6}, +{0x0000D8, 0x0000F8}, +{0x0000D9, 0x0000F9}, +{0x0000DA, 0x0000FA}, +{0x0000DB, 0x0000FB}, +{0x0000DC, 0x0000FC}, +{0x0000DD, 0x0000FD}, +{0x0000DE, 0x0000FE}, +{0x000100, 0x000101}, +{0x000102, 0x000103}, +{0x000104, 0x000105}, +{0x000106, 0x000107}, +{0x000108, 0x000109}, +{0x00010A, 0x00010B}, +{0x00010C, 0x00010D}, +{0x00010E, 0x00010F}, +{0x000110, 0x000111}, +{0x000112, 0x000113}, +{0x000114, 0x000115}, +{0x000116, 0x000117}, +{0x000118, 0x000119}, +{0x00011A, 0x00011B}, +{0x00011C, 0x00011D}, +{0x00011E, 0x00011F}, +{0x000120, 0x000121}, +{0x000122, 0x000123}, +{0x000124, 0x000125}, +{0x000126, 0x000127}, +{0x000128, 0x000129}, +{0x00012A, 0x00012B}, +{0x00012C, 0x00012D}, +{0x00012E, 0x00012F}, +{0x000130, 0x000069}, +{0x000132, 0x000133}, +{0x000134, 0x000135}, +{0x000136, 0x000137}, +{0x000139, 0x00013A}, +{0x00013B, 0x00013C}, +{0x00013D, 0x00013E}, +{0x00013F, 0x000140}, +{0x000141, 0x000142}, +{0x000143, 0x000144}, +{0x000145, 0x000146}, +{0x000147, 0x000148}, +{0x00014A, 0x00014B}, +{0x00014C, 0x00014D}, +{0x00014E, 0x00014F}, +{0x000150, 0x000151}, +{0x000152, 0x000153}, +{0x000154, 0x000155}, +{0x000156, 0x000157}, +{0x000158, 0x000159}, +{0x00015A, 0x00015B}, +{0x00015C, 0x00015D}, +{0x00015E, 0x00015F}, +{0x000160, 0x000161}, +{0x000162, 0x000163}, +{0x000164, 0x000165}, +{0x000166, 0x000167}, +{0x000168, 0x000169}, +{0x00016A, 0x00016B}, +{0x00016C, 0x00016D}, +{0x00016E, 0x00016F}, +{0x000170, 0x000171}, +{0x000172, 0x000173}, +{0x000174, 0x000175}, +{0x000176, 0x000177}, +{0x000178, 0x0000FF}, +{0x000179, 0x00017A}, +{0x00017B, 0x00017C}, +{0x00017D, 0x00017E}, +{0x000181, 0x000253}, +{0x000182, 0x000183}, +{0x000184, 0x000185}, +{0x000186, 0x000254}, +{0x000187, 0x000188}, +{0x000189, 0x000256}, +{0x00018A, 0x000257}, +{0x00018B, 0x00018C}, +{0x00018E, 0x0001DD}, +{0x00018F, 0x000259}, +{0x000190, 0x00025B}, +{0x000191, 0x000192}, +{0x000193, 0x000260}, +{0x000194, 0x000263}, +{0x000196, 0x000269}, +{0x000197, 0x000268}, +{0x000198, 0x000199}, +{0x00019C, 0x00026F}, +{0x00019D, 0x000272}, +{0x00019F, 0x000275}, +{0x0001A0, 0x0001A1}, +{0x0001A2, 0x0001A3}, +{0x0001A4, 0x0001A5}, +{0x0001A6, 0x000280}, +{0x0001A7, 0x0001A8}, +{0x0001A9, 0x000283}, +{0x0001AC, 0x0001AD}, +{0x0001AE, 0x000288}, +{0x0001AF, 0x0001B0}, +{0x0001B1, 0x00028A}, +{0x0001B2, 0x00028B}, +{0x0001B3, 0x0001B4}, +{0x0001B5, 0x0001B6}, +{0x0001B7, 0x000292}, +{0x0001B8, 0x0001B9}, +{0x0001BC, 0x0001BD}, +{0x0001C4, 0x0001C6}, +{0x0001C5, 0x0001C6}, +{0x0001C7, 0x0001C9}, +{0x0001C8, 0x0001C9}, +{0x0001CA, 0x0001CC}, +{0x0001CB, 0x0001CC}, +{0x0001CD, 0x0001CE}, +{0x0001CF, 0x0001D0}, +{0x0001D1, 0x0001D2}, +{0x0001D3, 0x0001D4}, +{0x0001D5, 0x0001D6}, +{0x0001D7, 0x0001D8}, +{0x0001D9, 0x0001DA}, +{0x0001DB, 0x0001DC}, +{0x0001DE, 0x0001DF}, +{0x0001E0, 0x0001E1}, +{0x0001E2, 0x0001E3}, +{0x0001E4, 0x0001E5}, +{0x0001E6, 0x0001E7}, +{0x0001E8, 0x0001E9}, +{0x0001EA, 0x0001EB}, +{0x0001EC, 0x0001ED}, +{0x0001EE, 0x0001EF}, +{0x0001F1, 0x0001F3}, +{0x0001F2, 0x0001F3}, +{0x0001F4, 0x0001F5}, +{0x0001F6, 0x000195}, +{0x0001F7, 0x0001BF}, +{0x0001F8, 0x0001F9}, +{0x0001FA, 0x0001FB}, +{0x0001FC, 0x0001FD}, +{0x0001FE, 0x0001FF}, +{0x000200, 0x000201}, +{0x000202, 0x000203}, +{0x000204, 0x000205}, +{0x000206, 0x000207}, +{0x000208, 0x000209}, +{0x00020A, 0x00020B}, +{0x00020C, 0x00020D}, +{0x00020E, 0x00020F}, +{0x000210, 0x000211}, +{0x000212, 0x000213}, +{0x000214, 0x000215}, +{0x000216, 0x000217}, +{0x000218, 0x000219}, +{0x00021A, 0x00021B}, +{0x00021C, 0x00021D}, +{0x00021E, 0x00021F}, +{0x000220, 0x00019E}, +{0x000222, 0x000223}, +{0x000224, 0x000225}, +{0x000226, 0x000227}, +{0x000228, 0x000229}, +{0x00022A, 0x00022B}, +{0x00022C, 0x00022D}, +{0x00022E, 0x00022F}, +{0x000230, 0x000231}, +{0x000232, 0x000233}, +{0x00023A, 0x002C65}, +{0x00023B, 0x00023C}, +{0x00023D, 0x00019A}, +{0x00023E, 0x002C66}, +{0x000241, 0x000242}, +{0x000243, 0x000180}, +{0x000244, 0x000289}, +{0x000245, 0x00028C}, +{0x000246, 0x000247}, +{0x000248, 0x000249}, +{0x00024A, 0x00024B}, +{0x00024C, 0x00024D}, +{0x00024E, 0x00024F}, +{0x000370, 0x000371}, +{0x000372, 0x000373}, +{0x000376, 0x000377}, +{0x00037F, 0x0003F3}, +{0x000386, 0x0003AC}, +{0x000388, 0x0003AD}, +{0x000389, 0x0003AE}, +{0x00038A, 0x0003AF}, +{0x00038C, 0x0003CC}, +{0x00038E, 0x0003CD}, +{0x00038F, 0x0003CE}, +{0x000391, 0x0003B1}, +{0x000392, 0x0003B2}, +{0x000393, 0x0003B3}, +{0x000394, 0x0003B4}, +{0x000395, 0x0003B5}, +{0x000396, 0x0003B6}, +{0x000397, 0x0003B7}, +{0x000398, 0x0003B8}, +{0x000399, 0x0003B9}, +{0x00039A, 0x0003BA}, +{0x00039B, 0x0003BB}, +{0x00039C, 0x0003BC}, +{0x00039D, 0x0003BD}, +{0x00039E, 0x0003BE}, +{0x00039F, 0x0003BF}, +{0x0003A0, 0x0003C0}, +{0x0003A1, 0x0003C1}, +{0x0003A3, 0x0003C3}, +{0x0003A4, 0x0003C4}, +{0x0003A5, 0x0003C5}, +{0x0003A6, 0x0003C6}, +{0x0003A7, 0x0003C7}, +{0x0003A8, 0x0003C8}, +{0x0003A9, 0x0003C9}, +{0x0003AA, 0x0003CA}, +{0x0003AB, 0x0003CB}, +{0x0003CF, 0x0003D7}, +{0x0003D8, 0x0003D9}, +{0x0003DA, 0x0003DB}, +{0x0003DC, 0x0003DD}, +{0x0003DE, 0x0003DF}, +{0x0003E0, 0x0003E1}, +{0x0003E2, 0x0003E3}, +{0x0003E4, 0x0003E5}, +{0x0003E6, 0x0003E7}, +{0x0003E8, 0x0003E9}, +{0x0003EA, 0x0003EB}, +{0x0003EC, 0x0003ED}, +{0x0003EE, 0x0003EF}, +{0x0003F4, 0x0003B8}, +{0x0003F7, 0x0003F8}, +{0x0003F9, 0x0003F2}, +{0x0003FA, 0x0003FB}, +{0x0003FD, 0x00037B}, +{0x0003FE, 0x00037C}, +{0x0003FF, 0x00037D}, +{0x000400, 0x000450}, +{0x000401, 0x000451}, +{0x000402, 0x000452}, +{0x000403, 0x000453}, +{0x000404, 0x000454}, +{0x000405, 0x000455}, +{0x000406, 0x000456}, +{0x000407, 0x000457}, +{0x000408, 0x000458}, +{0x000409, 0x000459}, +{0x00040A, 0x00045A}, +{0x00040B, 0x00045B}, +{0x00040C, 0x00045C}, +{0x00040D, 0x00045D}, +{0x00040E, 0x00045E}, +{0x00040F, 0x00045F}, +{0x000410, 0x000430}, +{0x000411, 0x000431}, +{0x000412, 0x000432}, +{0x000413, 0x000433}, +{0x000414, 0x000434}, +{0x000415, 0x000435}, +{0x000416, 0x000436}, +{0x000417, 0x000437}, +{0x000418, 0x000438}, +{0x000419, 0x000439}, +{0x00041A, 0x00043A}, +{0x00041B, 0x00043B}, +{0x00041C, 0x00043C}, +{0x00041D, 0x00043D}, +{0x00041E, 0x00043E}, +{0x00041F, 0x00043F}, +{0x000420, 0x000440}, +{0x000421, 0x000441}, +{0x000422, 0x000442}, +{0x000423, 0x000443}, +{0x000424, 0x000444}, +{0x000425, 0x000445}, +{0x000426, 0x000446}, +{0x000427, 0x000447}, +{0x000428, 0x000448}, +{0x000429, 0x000449}, +{0x00042A, 0x00044A}, +{0x00042B, 0x00044B}, +{0x00042C, 0x00044C}, +{0x00042D, 0x00044D}, +{0x00042E, 0x00044E}, +{0x00042F, 0x00044F}, +{0x000460, 0x000461}, +{0x000462, 0x000463}, +{0x000464, 0x000465}, +{0x000466, 0x000467}, +{0x000468, 0x000469}, +{0x00046A, 0x00046B}, +{0x00046C, 0x00046D}, +{0x00046E, 0x00046F}, +{0x000470, 0x000471}, +{0x000472, 0x000473}, +{0x000474, 0x000475}, +{0x000476, 0x000477}, +{0x000478, 0x000479}, +{0x00047A, 0x00047B}, +{0x00047C, 0x00047D}, +{0x00047E, 0x00047F}, +{0x000480, 0x000481}, +{0x00048A, 0x00048B}, +{0x00048C, 0x00048D}, +{0x00048E, 0x00048F}, +{0x000490, 0x000491}, +{0x000492, 0x000493}, +{0x000494, 0x000495}, +{0x000496, 0x000497}, +{0x000498, 0x000499}, +{0x00049A, 0x00049B}, +{0x00049C, 0x00049D}, +{0x00049E, 0x00049F}, +{0x0004A0, 0x0004A1}, +{0x0004A2, 0x0004A3}, +{0x0004A4, 0x0004A5}, +{0x0004A6, 0x0004A7}, +{0x0004A8, 0x0004A9}, +{0x0004AA, 0x0004AB}, +{0x0004AC, 0x0004AD}, +{0x0004AE, 0x0004AF}, +{0x0004B0, 0x0004B1}, +{0x0004B2, 0x0004B3}, +{0x0004B4, 0x0004B5}, +{0x0004B6, 0x0004B7}, +{0x0004B8, 0x0004B9}, +{0x0004BA, 0x0004BB}, +{0x0004BC, 0x0004BD}, +{0x0004BE, 0x0004BF}, +{0x0004C0, 0x0004CF}, +{0x0004C1, 0x0004C2}, +{0x0004C3, 0x0004C4}, +{0x0004C5, 0x0004C6}, +{0x0004C7, 0x0004C8}, +{0x0004C9, 0x0004CA}, +{0x0004CB, 0x0004CC}, +{0x0004CD, 0x0004CE}, +{0x0004D0, 0x0004D1}, +{0x0004D2, 0x0004D3}, +{0x0004D4, 0x0004D5}, +{0x0004D6, 0x0004D7}, +{0x0004D8, 0x0004D9}, +{0x0004DA, 0x0004DB}, +{0x0004DC, 0x0004DD}, +{0x0004DE, 0x0004DF}, +{0x0004E0, 0x0004E1}, +{0x0004E2, 0x0004E3}, +{0x0004E4, 0x0004E5}, +{0x0004E6, 0x0004E7}, +{0x0004E8, 0x0004E9}, +{0x0004EA, 0x0004EB}, +{0x0004EC, 0x0004ED}, +{0x0004EE, 0x0004EF}, +{0x0004F0, 0x0004F1}, +{0x0004F2, 0x0004F3}, +{0x0004F4, 0x0004F5}, +{0x0004F6, 0x0004F7}, +{0x0004F8, 0x0004F9}, +{0x0004FA, 0x0004FB}, +{0x0004FC, 0x0004FD}, +{0x0004FE, 0x0004FF}, +{0x000500, 0x000501}, +{0x000502, 0x000503}, +{0x000504, 0x000505}, +{0x000506, 0x000507}, +{0x000508, 0x000509}, +{0x00050A, 0x00050B}, +{0x00050C, 0x00050D}, +{0x00050E, 0x00050F}, +{0x000510, 0x000511}, +{0x000512, 0x000513}, +{0x000514, 0x000515}, +{0x000516, 0x000517}, +{0x000518, 0x000519}, +{0x00051A, 0x00051B}, +{0x00051C, 0x00051D}, +{0x00051E, 0x00051F}, +{0x000520, 0x000521}, +{0x000522, 0x000523}, +{0x000524, 0x000525}, +{0x000526, 0x000527}, +{0x000528, 0x000529}, +{0x00052A, 0x00052B}, +{0x00052C, 0x00052D}, +{0x00052E, 0x00052F}, +{0x000531, 0x000561}, +{0x000532, 0x000562}, +{0x000533, 0x000563}, +{0x000534, 0x000564}, +{0x000535, 0x000565}, +{0x000536, 0x000566}, +{0x000537, 0x000567}, +{0x000538, 0x000568}, +{0x000539, 0x000569}, +{0x00053A, 0x00056A}, +{0x00053B, 0x00056B}, +{0x00053C, 0x00056C}, +{0x00053D, 0x00056D}, +{0x00053E, 0x00056E}, +{0x00053F, 0x00056F}, +{0x000540, 0x000570}, +{0x000541, 0x000571}, +{0x000542, 0x000572}, +{0x000543, 0x000573}, +{0x000544, 0x000574}, +{0x000545, 0x000575}, +{0x000546, 0x000576}, +{0x000547, 0x000577}, +{0x000548, 0x000578}, +{0x000549, 0x000579}, +{0x00054A, 0x00057A}, +{0x00054B, 0x00057B}, +{0x00054C, 0x00057C}, +{0x00054D, 0x00057D}, +{0x00054E, 0x00057E}, +{0x00054F, 0x00057F}, +{0x000550, 0x000580}, +{0x000551, 0x000581}, +{0x000552, 0x000582}, +{0x000553, 0x000583}, +{0x000554, 0x000584}, +{0x000555, 0x000585}, +{0x000556, 0x000586}, +{0x0010A0, 0x002D00}, +{0x0010A1, 0x002D01}, +{0x0010A2, 0x002D02}, +{0x0010A3, 0x002D03}, +{0x0010A4, 0x002D04}, +{0x0010A5, 0x002D05}, +{0x0010A6, 0x002D06}, +{0x0010A7, 0x002D07}, +{0x0010A8, 0x002D08}, +{0x0010A9, 0x002D09}, +{0x0010AA, 0x002D0A}, +{0x0010AB, 0x002D0B}, +{0x0010AC, 0x002D0C}, +{0x0010AD, 0x002D0D}, +{0x0010AE, 0x002D0E}, +{0x0010AF, 0x002D0F}, +{0x0010B0, 0x002D10}, +{0x0010B1, 0x002D11}, +{0x0010B2, 0x002D12}, +{0x0010B3, 0x002D13}, +{0x0010B4, 0x002D14}, +{0x0010B5, 0x002D15}, +{0x0010B6, 0x002D16}, +{0x0010B7, 0x002D17}, +{0x0010B8, 0x002D18}, +{0x0010B9, 0x002D19}, +{0x0010BA, 0x002D1A}, +{0x0010BB, 0x002D1B}, +{0x0010BC, 0x002D1C}, +{0x0010BD, 0x002D1D}, +{0x0010BE, 0x002D1E}, +{0x0010BF, 0x002D1F}, +{0x0010C0, 0x002D20}, +{0x0010C1, 0x002D21}, +{0x0010C2, 0x002D22}, +{0x0010C3, 0x002D23}, +{0x0010C4, 0x002D24}, +{0x0010C5, 0x002D25}, +{0x0010C7, 0x002D27}, +{0x0010CD, 0x002D2D}, +{0x0013A0, 0x00AB70}, +{0x0013A1, 0x00AB71}, +{0x0013A2, 0x00AB72}, +{0x0013A3, 0x00AB73}, +{0x0013A4, 0x00AB74}, +{0x0013A5, 0x00AB75}, +{0x0013A6, 0x00AB76}, +{0x0013A7, 0x00AB77}, +{0x0013A8, 0x00AB78}, +{0x0013A9, 0x00AB79}, +{0x0013AA, 0x00AB7A}, +{0x0013AB, 0x00AB7B}, +{0x0013AC, 0x00AB7C}, +{0x0013AD, 0x00AB7D}, +{0x0013AE, 0x00AB7E}, +{0x0013AF, 0x00AB7F}, +{0x0013B0, 0x00AB80}, +{0x0013B1, 0x00AB81}, +{0x0013B2, 0x00AB82}, +{0x0013B3, 0x00AB83}, +{0x0013B4, 0x00AB84}, +{0x0013B5, 0x00AB85}, +{0x0013B6, 0x00AB86}, +{0x0013B7, 0x00AB87}, +{0x0013B8, 0x00AB88}, +{0x0013B9, 0x00AB89}, +{0x0013BA, 0x00AB8A}, +{0x0013BB, 0x00AB8B}, +{0x0013BC, 0x00AB8C}, +{0x0013BD, 0x00AB8D}, +{0x0013BE, 0x00AB8E}, +{0x0013BF, 0x00AB8F}, +{0x0013C0, 0x00AB90}, +{0x0013C1, 0x00AB91}, +{0x0013C2, 0x00AB92}, +{0x0013C3, 0x00AB93}, +{0x0013C4, 0x00AB94}, +{0x0013C5, 0x00AB95}, +{0x0013C6, 0x00AB96}, +{0x0013C7, 0x00AB97}, +{0x0013C8, 0x00AB98}, +{0x0013C9, 0x00AB99}, +{0x0013CA, 0x00AB9A}, +{0x0013CB, 0x00AB9B}, +{0x0013CC, 0x00AB9C}, +{0x0013CD, 0x00AB9D}, +{0x0013CE, 0x00AB9E}, +{0x0013CF, 0x00AB9F}, +{0x0013D0, 0x00ABA0}, +{0x0013D1, 0x00ABA1}, +{0x0013D2, 0x00ABA2}, +{0x0013D3, 0x00ABA3}, +{0x0013D4, 0x00ABA4}, +{0x0013D5, 0x00ABA5}, +{0x0013D6, 0x00ABA6}, +{0x0013D7, 0x00ABA7}, +{0x0013D8, 0x00ABA8}, +{0x0013D9, 0x00ABA9}, +{0x0013DA, 0x00ABAA}, +{0x0013DB, 0x00ABAB}, +{0x0013DC, 0x00ABAC}, +{0x0013DD, 0x00ABAD}, +{0x0013DE, 0x00ABAE}, +{0x0013DF, 0x00ABAF}, +{0x0013E0, 0x00ABB0}, +{0x0013E1, 0x00ABB1}, +{0x0013E2, 0x00ABB2}, +{0x0013E3, 0x00ABB3}, +{0x0013E4, 0x00ABB4}, +{0x0013E5, 0x00ABB5}, +{0x0013E6, 0x00ABB6}, +{0x0013E7, 0x00ABB7}, +{0x0013E8, 0x00ABB8}, +{0x0013E9, 0x00ABB9}, +{0x0013EA, 0x00ABBA}, +{0x0013EB, 0x00ABBB}, +{0x0013EC, 0x00ABBC}, +{0x0013ED, 0x00ABBD}, +{0x0013EE, 0x00ABBE}, +{0x0013EF, 0x00ABBF}, +{0x0013F0, 0x0013F8}, +{0x0013F1, 0x0013F9}, +{0x0013F2, 0x0013FA}, +{0x0013F3, 0x0013FB}, +{0x0013F4, 0x0013FC}, +{0x0013F5, 0x0013FD}, +{0x001C90, 0x0010D0}, +{0x001C91, 0x0010D1}, +{0x001C92, 0x0010D2}, +{0x001C93, 0x0010D3}, +{0x001C94, 0x0010D4}, +{0x001C95, 0x0010D5}, +{0x001C96, 0x0010D6}, +{0x001C97, 0x0010D7}, +{0x001C98, 0x0010D8}, +{0x001C99, 0x0010D9}, +{0x001C9A, 0x0010DA}, +{0x001C9B, 0x0010DB}, +{0x001C9C, 0x0010DC}, +{0x001C9D, 0x0010DD}, +{0x001C9E, 0x0010DE}, +{0x001C9F, 0x0010DF}, +{0x001CA0, 0x0010E0}, +{0x001CA1, 0x0010E1}, +{0x001CA2, 0x0010E2}, +{0x001CA3, 0x0010E3}, +{0x001CA4, 0x0010E4}, +{0x001CA5, 0x0010E5}, +{0x001CA6, 0x0010E6}, +{0x001CA7, 0x0010E7}, +{0x001CA8, 0x0010E8}, +{0x001CA9, 0x0010E9}, +{0x001CAA, 0x0010EA}, +{0x001CAB, 0x0010EB}, +{0x001CAC, 0x0010EC}, +{0x001CAD, 0x0010ED}, +{0x001CAE, 0x0010EE}, +{0x001CAF, 0x0010EF}, +{0x001CB0, 0x0010F0}, +{0x001CB1, 0x0010F1}, +{0x001CB2, 0x0010F2}, +{0x001CB3, 0x0010F3}, +{0x001CB4, 0x0010F4}, +{0x001CB5, 0x0010F5}, +{0x001CB6, 0x0010F6}, +{0x001CB7, 0x0010F7}, +{0x001CB8, 0x0010F8}, +{0x001CB9, 0x0010F9}, +{0x001CBA, 0x0010FA}, +{0x001CBD, 0x0010FD}, +{0x001CBE, 0x0010FE}, +{0x001CBF, 0x0010FF}, +{0x001E00, 0x001E01}, +{0x001E02, 0x001E03}, +{0x001E04, 0x001E05}, +{0x001E06, 0x001E07}, +{0x001E08, 0x001E09}, +{0x001E0A, 0x001E0B}, +{0x001E0C, 0x001E0D}, +{0x001E0E, 0x001E0F}, +{0x001E10, 0x001E11}, +{0x001E12, 0x001E13}, +{0x001E14, 0x001E15}, +{0x001E16, 0x001E17}, +{0x001E18, 0x001E19}, +{0x001E1A, 0x001E1B}, +{0x001E1C, 0x001E1D}, +{0x001E1E, 0x001E1F}, +{0x001E20, 0x001E21}, +{0x001E22, 0x001E23}, +{0x001E24, 0x001E25}, +{0x001E26, 0x001E27}, +{0x001E28, 0x001E29}, +{0x001E2A, 0x001E2B}, +{0x001E2C, 0x001E2D}, +{0x001E2E, 0x001E2F}, +{0x001E30, 0x001E31}, +{0x001E32, 0x001E33}, +{0x001E34, 0x001E35}, +{0x001E36, 0x001E37}, +{0x001E38, 0x001E39}, +{0x001E3A, 0x001E3B}, +{0x001E3C, 0x001E3D}, +{0x001E3E, 0x001E3F}, +{0x001E40, 0x001E41}, +{0x001E42, 0x001E43}, +{0x001E44, 0x001E45}, +{0x001E46, 0x001E47}, +{0x001E48, 0x001E49}, +{0x001E4A, 0x001E4B}, +{0x001E4C, 0x001E4D}, +{0x001E4E, 0x001E4F}, +{0x001E50, 0x001E51}, +{0x001E52, 0x001E53}, +{0x001E54, 0x001E55}, +{0x001E56, 0x001E57}, +{0x001E58, 0x001E59}, +{0x001E5A, 0x001E5B}, +{0x001E5C, 0x001E5D}, +{0x001E5E, 0x001E5F}, +{0x001E60, 0x001E61}, +{0x001E62, 0x001E63}, +{0x001E64, 0x001E65}, +{0x001E66, 0x001E67}, +{0x001E68, 0x001E69}, +{0x001E6A, 0x001E6B}, +{0x001E6C, 0x001E6D}, +{0x001E6E, 0x001E6F}, +{0x001E70, 0x001E71}, +{0x001E72, 0x001E73}, +{0x001E74, 0x001E75}, +{0x001E76, 0x001E77}, +{0x001E78, 0x001E79}, +{0x001E7A, 0x001E7B}, +{0x001E7C, 0x001E7D}, +{0x001E7E, 0x001E7F}, +{0x001E80, 0x001E81}, +{0x001E82, 0x001E83}, +{0x001E84, 0x001E85}, +{0x001E86, 0x001E87}, +{0x001E88, 0x001E89}, +{0x001E8A, 0x001E8B}, +{0x001E8C, 0x001E8D}, +{0x001E8E, 0x001E8F}, +{0x001E90, 0x001E91}, +{0x001E92, 0x001E93}, +{0x001E94, 0x001E95}, +{0x001E9E, 0x0000DF}, +{0x001EA0, 0x001EA1}, +{0x001EA2, 0x001EA3}, +{0x001EA4, 0x001EA5}, +{0x001EA6, 0x001EA7}, +{0x001EA8, 0x001EA9}, +{0x001EAA, 0x001EAB}, +{0x001EAC, 0x001EAD}, +{0x001EAE, 0x001EAF}, +{0x001EB0, 0x001EB1}, +{0x001EB2, 0x001EB3}, +{0x001EB4, 0x001EB5}, +{0x001EB6, 0x001EB7}, +{0x001EB8, 0x001EB9}, +{0x001EBA, 0x001EBB}, +{0x001EBC, 0x001EBD}, +{0x001EBE, 0x001EBF}, +{0x001EC0, 0x001EC1}, +{0x001EC2, 0x001EC3}, +{0x001EC4, 0x001EC5}, +{0x001EC6, 0x001EC7}, +{0x001EC8, 0x001EC9}, +{0x001ECA, 0x001ECB}, +{0x001ECC, 0x001ECD}, +{0x001ECE, 0x001ECF}, +{0x001ED0, 0x001ED1}, +{0x001ED2, 0x001ED3}, +{0x001ED4, 0x001ED5}, +{0x001ED6, 0x001ED7}, +{0x001ED8, 0x001ED9}, +{0x001EDA, 0x001EDB}, +{0x001EDC, 0x001EDD}, +{0x001EDE, 0x001EDF}, +{0x001EE0, 0x001EE1}, +{0x001EE2, 0x001EE3}, +{0x001EE4, 0x001EE5}, +{0x001EE6, 0x001EE7}, +{0x001EE8, 0x001EE9}, +{0x001EEA, 0x001EEB}, +{0x001EEC, 0x001EED}, +{0x001EEE, 0x001EEF}, +{0x001EF0, 0x001EF1}, +{0x001EF2, 0x001EF3}, +{0x001EF4, 0x001EF5}, +{0x001EF6, 0x001EF7}, +{0x001EF8, 0x001EF9}, +{0x001EFA, 0x001EFB}, +{0x001EFC, 0x001EFD}, +{0x001EFE, 0x001EFF}, +{0x001F08, 0x001F00}, +{0x001F09, 0x001F01}, +{0x001F0A, 0x001F02}, +{0x001F0B, 0x001F03}, +{0x001F0C, 0x001F04}, +{0x001F0D, 0x001F05}, +{0x001F0E, 0x001F06}, +{0x001F0F, 0x001F07}, +{0x001F18, 0x001F10}, +{0x001F19, 0x001F11}, +{0x001F1A, 0x001F12}, +{0x001F1B, 0x001F13}, +{0x001F1C, 0x001F14}, +{0x001F1D, 0x001F15}, +{0x001F28, 0x001F20}, +{0x001F29, 0x001F21}, +{0x001F2A, 0x001F22}, +{0x001F2B, 0x001F23}, +{0x001F2C, 0x001F24}, +{0x001F2D, 0x001F25}, +{0x001F2E, 0x001F26}, +{0x001F2F, 0x001F27}, +{0x001F38, 0x001F30}, +{0x001F39, 0x001F31}, +{0x001F3A, 0x001F32}, +{0x001F3B, 0x001F33}, +{0x001F3C, 0x001F34}, +{0x001F3D, 0x001F35}, +{0x001F3E, 0x001F36}, +{0x001F3F, 0x001F37}, +{0x001F48, 0x001F40}, +{0x001F49, 0x001F41}, +{0x001F4A, 0x001F42}, +{0x001F4B, 0x001F43}, +{0x001F4C, 0x001F44}, +{0x001F4D, 0x001F45}, +{0x001F59, 0x001F51}, +{0x001F5B, 0x001F53}, +{0x001F5D, 0x001F55}, +{0x001F5F, 0x001F57}, +{0x001F68, 0x001F60}, +{0x001F69, 0x001F61}, +{0x001F6A, 0x001F62}, +{0x001F6B, 0x001F63}, +{0x001F6C, 0x001F64}, +{0x001F6D, 0x001F65}, +{0x001F6E, 0x001F66}, +{0x001F6F, 0x001F67}, +{0x001F88, 0x001F80}, +{0x001F89, 0x001F81}, +{0x001F8A, 0x001F82}, +{0x001F8B, 0x001F83}, +{0x001F8C, 0x001F84}, +{0x001F8D, 0x001F85}, +{0x001F8E, 0x001F86}, +{0x001F8F, 0x001F87}, +{0x001F98, 0x001F90}, +{0x001F99, 0x001F91}, +{0x001F9A, 0x001F92}, +{0x001F9B, 0x001F93}, +{0x001F9C, 0x001F94}, +{0x001F9D, 0x001F95}, +{0x001F9E, 0x001F96}, +{0x001F9F, 0x001F97}, +{0x001FA8, 0x001FA0}, +{0x001FA9, 0x001FA1}, +{0x001FAA, 0x001FA2}, +{0x001FAB, 0x001FA3}, +{0x001FAC, 0x001FA4}, +{0x001FAD, 0x001FA5}, +{0x001FAE, 0x001FA6}, +{0x001FAF, 0x001FA7}, +{0x001FB8, 0x001FB0}, +{0x001FB9, 0x001FB1}, +{0x001FBA, 0x001F70}, +{0x001FBB, 0x001F71}, +{0x001FBC, 0x001FB3}, +{0x001FC8, 0x001F72}, +{0x001FC9, 0x001F73}, +{0x001FCA, 0x001F74}, +{0x001FCB, 0x001F75}, +{0x001FCC, 0x001FC3}, +{0x001FD8, 0x001FD0}, +{0x001FD9, 0x001FD1}, +{0x001FDA, 0x001F76}, +{0x001FDB, 0x001F77}, +{0x001FE8, 0x001FE0}, +{0x001FE9, 0x001FE1}, +{0x001FEA, 0x001F7A}, +{0x001FEB, 0x001F7B}, +{0x001FEC, 0x001FE5}, +{0x001FF8, 0x001F78}, +{0x001FF9, 0x001F79}, +{0x001FFA, 0x001F7C}, +{0x001FFB, 0x001F7D}, +{0x001FFC, 0x001FF3}, +{0x002126, 0x0003C9}, +{0x00212A, 0x00006B}, +{0x00212B, 0x0000E5}, +{0x002132, 0x00214E}, +{0x002160, 0x002170}, +{0x002161, 0x002171}, +{0x002162, 0x002172}, +{0x002163, 0x002173}, +{0x002164, 0x002174}, +{0x002165, 0x002175}, +{0x002166, 0x002176}, +{0x002167, 0x002177}, +{0x002168, 0x002178}, +{0x002169, 0x002179}, +{0x00216A, 0x00217A}, +{0x00216B, 0x00217B}, +{0x00216C, 0x00217C}, +{0x00216D, 0x00217D}, +{0x00216E, 0x00217E}, +{0x00216F, 0x00217F}, +{0x002183, 0x002184}, +{0x0024B6, 0x0024D0}, +{0x0024B7, 0x0024D1}, +{0x0024B8, 0x0024D2}, +{0x0024B9, 0x0024D3}, +{0x0024BA, 0x0024D4}, +{0x0024BB, 0x0024D5}, +{0x0024BC, 0x0024D6}, +{0x0024BD, 0x0024D7}, +{0x0024BE, 0x0024D8}, +{0x0024BF, 0x0024D9}, +{0x0024C0, 0x0024DA}, +{0x0024C1, 0x0024DB}, +{0x0024C2, 0x0024DC}, +{0x0024C3, 0x0024DD}, +{0x0024C4, 0x0024DE}, +{0x0024C5, 0x0024DF}, +{0x0024C6, 0x0024E0}, +{0x0024C7, 0x0024E1}, +{0x0024C8, 0x0024E2}, +{0x0024C9, 0x0024E3}, +{0x0024CA, 0x0024E4}, +{0x0024CB, 0x0024E5}, +{0x0024CC, 0x0024E6}, +{0x0024CD, 0x0024E7}, +{0x0024CE, 0x0024E8}, +{0x0024CF, 0x0024E9}, +{0x002C00, 0x002C30}, +{0x002C01, 0x002C31}, +{0x002C02, 0x002C32}, +{0x002C03, 0x002C33}, +{0x002C04, 0x002C34}, +{0x002C05, 0x002C35}, +{0x002C06, 0x002C36}, +{0x002C07, 0x002C37}, +{0x002C08, 0x002C38}, +{0x002C09, 0x002C39}, +{0x002C0A, 0x002C3A}, +{0x002C0B, 0x002C3B}, +{0x002C0C, 0x002C3C}, +{0x002C0D, 0x002C3D}, +{0x002C0E, 0x002C3E}, +{0x002C0F, 0x002C3F}, +{0x002C10, 0x002C40}, +{0x002C11, 0x002C41}, +{0x002C12, 0x002C42}, +{0x002C13, 0x002C43}, +{0x002C14, 0x002C44}, +{0x002C15, 0x002C45}, +{0x002C16, 0x002C46}, +{0x002C17, 0x002C47}, +{0x002C18, 0x002C48}, +{0x002C19, 0x002C49}, +{0x002C1A, 0x002C4A}, +{0x002C1B, 0x002C4B}, +{0x002C1C, 0x002C4C}, +{0x002C1D, 0x002C4D}, +{0x002C1E, 0x002C4E}, +{0x002C1F, 0x002C4F}, +{0x002C20, 0x002C50}, +{0x002C21, 0x002C51}, +{0x002C22, 0x002C52}, +{0x002C23, 0x002C53}, +{0x002C24, 0x002C54}, +{0x002C25, 0x002C55}, +{0x002C26, 0x002C56}, +{0x002C27, 0x002C57}, +{0x002C28, 0x002C58}, +{0x002C29, 0x002C59}, +{0x002C2A, 0x002C5A}, +{0x002C2B, 0x002C5B}, +{0x002C2C, 0x002C5C}, +{0x002C2D, 0x002C5D}, +{0x002C2E, 0x002C5E}, +{0x002C60, 0x002C61}, +{0x002C62, 0x00026B}, +{0x002C63, 0x001D7D}, +{0x002C64, 0x00027D}, +{0x002C67, 0x002C68}, +{0x002C69, 0x002C6A}, +{0x002C6B, 0x002C6C}, +{0x002C6D, 0x000251}, +{0x002C6E, 0x000271}, +{0x002C6F, 0x000250}, +{0x002C70, 0x000252}, +{0x002C72, 0x002C73}, +{0x002C75, 0x002C76}, +{0x002C7E, 0x00023F}, +{0x002C7F, 0x000240}, +{0x002C80, 0x002C81}, +{0x002C82, 0x002C83}, +{0x002C84, 0x002C85}, +{0x002C86, 0x002C87}, +{0x002C88, 0x002C89}, +{0x002C8A, 0x002C8B}, +{0x002C8C, 0x002C8D}, +{0x002C8E, 0x002C8F}, +{0x002C90, 0x002C91}, +{0x002C92, 0x002C93}, +{0x002C94, 0x002C95}, +{0x002C96, 0x002C97}, +{0x002C98, 0x002C99}, +{0x002C9A, 0x002C9B}, +{0x002C9C, 0x002C9D}, +{0x002C9E, 0x002C9F}, +{0x002CA0, 0x002CA1}, +{0x002CA2, 0x002CA3}, +{0x002CA4, 0x002CA5}, +{0x002CA6, 0x002CA7}, +{0x002CA8, 0x002CA9}, +{0x002CAA, 0x002CAB}, +{0x002CAC, 0x002CAD}, +{0x002CAE, 0x002CAF}, +{0x002CB0, 0x002CB1}, +{0x002CB2, 0x002CB3}, +{0x002CB4, 0x002CB5}, +{0x002CB6, 0x002CB7}, +{0x002CB8, 0x002CB9}, +{0x002CBA, 0x002CBB}, +{0x002CBC, 0x002CBD}, +{0x002CBE, 0x002CBF}, +{0x002CC0, 0x002CC1}, +{0x002CC2, 0x002CC3}, +{0x002CC4, 0x002CC5}, +{0x002CC6, 0x002CC7}, +{0x002CC8, 0x002CC9}, +{0x002CCA, 0x002CCB}, +{0x002CCC, 0x002CCD}, +{0x002CCE, 0x002CCF}, +{0x002CD0, 0x002CD1}, +{0x002CD2, 0x002CD3}, +{0x002CD4, 0x002CD5}, +{0x002CD6, 0x002CD7}, +{0x002CD8, 0x002CD9}, +{0x002CDA, 0x002CDB}, +{0x002CDC, 0x002CDD}, +{0x002CDE, 0x002CDF}, +{0x002CE0, 0x002CE1}, +{0x002CE2, 0x002CE3}, +{0x002CEB, 0x002CEC}, +{0x002CED, 0x002CEE}, +{0x002CF2, 0x002CF3}, +{0x00A640, 0x00A641}, +{0x00A642, 0x00A643}, +{0x00A644, 0x00A645}, +{0x00A646, 0x00A647}, +{0x00A648, 0x00A649}, +{0x00A64A, 0x00A64B}, +{0x00A64C, 0x00A64D}, +{0x00A64E, 0x00A64F}, +{0x00A650, 0x00A651}, +{0x00A652, 0x00A653}, +{0x00A654, 0x00A655}, +{0x00A656, 0x00A657}, +{0x00A658, 0x00A659}, +{0x00A65A, 0x00A65B}, +{0x00A65C, 0x00A65D}, +{0x00A65E, 0x00A65F}, +{0x00A660, 0x00A661}, +{0x00A662, 0x00A663}, +{0x00A664, 0x00A665}, +{0x00A666, 0x00A667}, +{0x00A668, 0x00A669}, +{0x00A66A, 0x00A66B}, +{0x00A66C, 0x00A66D}, +{0x00A680, 0x00A681}, +{0x00A682, 0x00A683}, +{0x00A684, 0x00A685}, +{0x00A686, 0x00A687}, +{0x00A688, 0x00A689}, +{0x00A68A, 0x00A68B}, +{0x00A68C, 0x00A68D}, +{0x00A68E, 0x00A68F}, +{0x00A690, 0x00A691}, +{0x00A692, 0x00A693}, +{0x00A694, 0x00A695}, +{0x00A696, 0x00A697}, +{0x00A698, 0x00A699}, +{0x00A69A, 0x00A69B}, +{0x00A722, 0x00A723}, +{0x00A724, 0x00A725}, +{0x00A726, 0x00A727}, +{0x00A728, 0x00A729}, +{0x00A72A, 0x00A72B}, +{0x00A72C, 0x00A72D}, +{0x00A72E, 0x00A72F}, +{0x00A732, 0x00A733}, +{0x00A734, 0x00A735}, +{0x00A736, 0x00A737}, +{0x00A738, 0x00A739}, +{0x00A73A, 0x00A73B}, +{0x00A73C, 0x00A73D}, +{0x00A73E, 0x00A73F}, +{0x00A740, 0x00A741}, +{0x00A742, 0x00A743}, +{0x00A744, 0x00A745}, +{0x00A746, 0x00A747}, +{0x00A748, 0x00A749}, +{0x00A74A, 0x00A74B}, +{0x00A74C, 0x00A74D}, +{0x00A74E, 0x00A74F}, +{0x00A750, 0x00A751}, +{0x00A752, 0x00A753}, +{0x00A754, 0x00A755}, +{0x00A756, 0x00A757}, +{0x00A758, 0x00A759}, +{0x00A75A, 0x00A75B}, +{0x00A75C, 0x00A75D}, +{0x00A75E, 0x00A75F}, +{0x00A760, 0x00A761}, +{0x00A762, 0x00A763}, +{0x00A764, 0x00A765}, +{0x00A766, 0x00A767}, +{0x00A768, 0x00A769}, +{0x00A76A, 0x00A76B}, +{0x00A76C, 0x00A76D}, +{0x00A76E, 0x00A76F}, +{0x00A779, 0x00A77A}, +{0x00A77B, 0x00A77C}, +{0x00A77D, 0x001D79}, +{0x00A77E, 0x00A77F}, +{0x00A780, 0x00A781}, +{0x00A782, 0x00A783}, +{0x00A784, 0x00A785}, +{0x00A786, 0x00A787}, +{0x00A78B, 0x00A78C}, +{0x00A78D, 0x000265}, +{0x00A790, 0x00A791}, +{0x00A792, 0x00A793}, +{0x00A796, 0x00A797}, +{0x00A798, 0x00A799}, +{0x00A79A, 0x00A79B}, +{0x00A79C, 0x00A79D}, +{0x00A79E, 0x00A79F}, +{0x00A7A0, 0x00A7A1}, +{0x00A7A2, 0x00A7A3}, +{0x00A7A4, 0x00A7A5}, +{0x00A7A6, 0x00A7A7}, +{0x00A7A8, 0x00A7A9}, +{0x00A7AA, 0x000266}, +{0x00A7AB, 0x00025C}, +{0x00A7AC, 0x000261}, +{0x00A7AD, 0x00026C}, +{0x00A7AE, 0x00026A}, +{0x00A7B0, 0x00029E}, +{0x00A7B1, 0x000287}, +{0x00A7B2, 0x00029D}, +{0x00A7B3, 0x00AB53}, +{0x00A7B4, 0x00A7B5}, +{0x00A7B6, 0x00A7B7}, +{0x00A7B8, 0x00A7B9}, +{0x00A7BA, 0x00A7BB}, +{0x00A7BC, 0x00A7BD}, +{0x00A7BE, 0x00A7BF}, +{0x00A7C2, 0x00A7C3}, +{0x00A7C4, 0x00A794}, +{0x00A7C5, 0x000282}, +{0x00A7C6, 0x001D8E}, +{0x00A7C7, 0x00A7C8}, +{0x00A7C9, 0x00A7CA}, +{0x00A7F5, 0x00A7F6}, +{0x00FF21, 0x00FF41}, +{0x00FF22, 0x00FF42}, +{0x00FF23, 0x00FF43}, +{0x00FF24, 0x00FF44}, +{0x00FF25, 0x00FF45}, +{0x00FF26, 0x00FF46}, +{0x00FF27, 0x00FF47}, +{0x00FF28, 0x00FF48}, +{0x00FF29, 0x00FF49}, +{0x00FF2A, 0x00FF4A}, +{0x00FF2B, 0x00FF4B}, +{0x00FF2C, 0x00FF4C}, +{0x00FF2D, 0x00FF4D}, +{0x00FF2E, 0x00FF4E}, +{0x00FF2F, 0x00FF4F}, +{0x00FF30, 0x00FF50}, +{0x00FF31, 0x00FF51}, +{0x00FF32, 0x00FF52}, +{0x00FF33, 0x00FF53}, +{0x00FF34, 0x00FF54}, +{0x00FF35, 0x00FF55}, +{0x00FF36, 0x00FF56}, +{0x00FF37, 0x00FF57}, +{0x00FF38, 0x00FF58}, +{0x00FF39, 0x00FF59}, +{0x00FF3A, 0x00FF5A}, +{0x010400, 0x010428}, +{0x010401, 0x010429}, +{0x010402, 0x01042A}, +{0x010403, 0x01042B}, +{0x010404, 0x01042C}, +{0x010405, 0x01042D}, +{0x010406, 0x01042E}, +{0x010407, 0x01042F}, +{0x010408, 0x010430}, +{0x010409, 0x010431}, +{0x01040A, 0x010432}, +{0x01040B, 0x010433}, +{0x01040C, 0x010434}, +{0x01040D, 0x010435}, +{0x01040E, 0x010436}, +{0x01040F, 0x010437}, +{0x010410, 0x010438}, +{0x010411, 0x010439}, +{0x010412, 0x01043A}, +{0x010413, 0x01043B}, +{0x010414, 0x01043C}, +{0x010415, 0x01043D}, +{0x010416, 0x01043E}, +{0x010417, 0x01043F}, +{0x010418, 0x010440}, +{0x010419, 0x010441}, +{0x01041A, 0x010442}, +{0x01041B, 0x010443}, +{0x01041C, 0x010444}, +{0x01041D, 0x010445}, +{0x01041E, 0x010446}, +{0x01041F, 0x010447}, +{0x010420, 0x010448}, +{0x010421, 0x010449}, +{0x010422, 0x01044A}, +{0x010423, 0x01044B}, +{0x010424, 0x01044C}, +{0x010425, 0x01044D}, +{0x010426, 0x01044E}, +{0x010427, 0x01044F}, +{0x0104B0, 0x0104D8}, +{0x0104B1, 0x0104D9}, +{0x0104B2, 0x0104DA}, +{0x0104B3, 0x0104DB}, +{0x0104B4, 0x0104DC}, +{0x0104B5, 0x0104DD}, +{0x0104B6, 0x0104DE}, +{0x0104B7, 0x0104DF}, +{0x0104B8, 0x0104E0}, +{0x0104B9, 0x0104E1}, +{0x0104BA, 0x0104E2}, +{0x0104BB, 0x0104E3}, +{0x0104BC, 0x0104E4}, +{0x0104BD, 0x0104E5}, +{0x0104BE, 0x0104E6}, +{0x0104BF, 0x0104E7}, +{0x0104C0, 0x0104E8}, +{0x0104C1, 0x0104E9}, +{0x0104C2, 0x0104EA}, +{0x0104C3, 0x0104EB}, +{0x0104C4, 0x0104EC}, +{0x0104C5, 0x0104ED}, +{0x0104C6, 0x0104EE}, +{0x0104C7, 0x0104EF}, +{0x0104C8, 0x0104F0}, +{0x0104C9, 0x0104F1}, +{0x0104CA, 0x0104F2}, +{0x0104CB, 0x0104F3}, +{0x0104CC, 0x0104F4}, +{0x0104CD, 0x0104F5}, +{0x0104CE, 0x0104F6}, +{0x0104CF, 0x0104F7}, +{0x0104D0, 0x0104F8}, +{0x0104D1, 0x0104F9}, +{0x0104D2, 0x0104FA}, +{0x0104D3, 0x0104FB}, +{0x010C80, 0x010CC0}, +{0x010C81, 0x010CC1}, +{0x010C82, 0x010CC2}, +{0x010C83, 0x010CC3}, +{0x010C84, 0x010CC4}, +{0x010C85, 0x010CC5}, +{0x010C86, 0x010CC6}, +{0x010C87, 0x010CC7}, +{0x010C88, 0x010CC8}, +{0x010C89, 0x010CC9}, +{0x010C8A, 0x010CCA}, +{0x010C8B, 0x010CCB}, +{0x010C8C, 0x010CCC}, +{0x010C8D, 0x010CCD}, +{0x010C8E, 0x010CCE}, +{0x010C8F, 0x010CCF}, +{0x010C90, 0x010CD0}, +{0x010C91, 0x010CD1}, +{0x010C92, 0x010CD2}, +{0x010C93, 0x010CD3}, +{0x010C94, 0x010CD4}, +{0x010C95, 0x010CD5}, +{0x010C96, 0x010CD6}, +{0x010C97, 0x010CD7}, +{0x010C98, 0x010CD8}, +{0x010C99, 0x010CD9}, +{0x010C9A, 0x010CDA}, +{0x010C9B, 0x010CDB}, +{0x010C9C, 0x010CDC}, +{0x010C9D, 0x010CDD}, +{0x010C9E, 0x010CDE}, +{0x010C9F, 0x010CDF}, +{0x010CA0, 0x010CE0}, +{0x010CA1, 0x010CE1}, +{0x010CA2, 0x010CE2}, +{0x010CA3, 0x010CE3}, +{0x010CA4, 0x010CE4}, +{0x010CA5, 0x010CE5}, +{0x010CA6, 0x010CE6}, +{0x010CA7, 0x010CE7}, +{0x010CA8, 0x010CE8}, +{0x010CA9, 0x010CE9}, +{0x010CAA, 0x010CEA}, +{0x010CAB, 0x010CEB}, +{0x010CAC, 0x010CEC}, +{0x010CAD, 0x010CED}, +{0x010CAE, 0x010CEE}, +{0x010CAF, 0x010CEF}, +{0x010CB0, 0x010CF0}, +{0x010CB1, 0x010CF1}, +{0x010CB2, 0x010CF2}, +{0x0118A0, 0x0118C0}, +{0x0118A1, 0x0118C1}, +{0x0118A2, 0x0118C2}, +{0x0118A3, 0x0118C3}, +{0x0118A4, 0x0118C4}, +{0x0118A5, 0x0118C5}, +{0x0118A6, 0x0118C6}, +{0x0118A7, 0x0118C7}, +{0x0118A8, 0x0118C8}, +{0x0118A9, 0x0118C9}, +{0x0118AA, 0x0118CA}, +{0x0118AB, 0x0118CB}, +{0x0118AC, 0x0118CC}, +{0x0118AD, 0x0118CD}, +{0x0118AE, 0x0118CE}, +{0x0118AF, 0x0118CF}, +{0x0118B0, 0x0118D0}, +{0x0118B1, 0x0118D1}, +{0x0118B2, 0x0118D2}, +{0x0118B3, 0x0118D3}, +{0x0118B4, 0x0118D4}, +{0x0118B5, 0x0118D5}, +{0x0118B6, 0x0118D6}, +{0x0118B7, 0x0118D7}, +{0x0118B8, 0x0118D8}, +{0x0118B9, 0x0118D9}, +{0x0118BA, 0x0118DA}, +{0x0118BB, 0x0118DB}, +{0x0118BC, 0x0118DC}, +{0x0118BD, 0x0118DD}, +{0x0118BE, 0x0118DE}, +{0x0118BF, 0x0118DF}, +{0x016E40, 0x016E60}, +{0x016E41, 0x016E61}, +{0x016E42, 0x016E62}, +{0x016E43, 0x016E63}, +{0x016E44, 0x016E64}, +{0x016E45, 0x016E65}, +{0x016E46, 0x016E66}, +{0x016E47, 0x016E67}, +{0x016E48, 0x016E68}, +{0x016E49, 0x016E69}, +{0x016E4A, 0x016E6A}, +{0x016E4B, 0x016E6B}, +{0x016E4C, 0x016E6C}, +{0x016E4D, 0x016E6D}, +{0x016E4E, 0x016E6E}, +{0x016E4F, 0x016E6F}, +{0x016E50, 0x016E70}, +{0x016E51, 0x016E71}, +{0x016E52, 0x016E72}, +{0x016E53, 0x016E73}, +{0x016E54, 0x016E74}, +{0x016E55, 0x016E75}, +{0x016E56, 0x016E76}, +{0x016E57, 0x016E77}, +{0x016E58, 0x016E78}, +{0x016E59, 0x016E79}, +{0x016E5A, 0x016E7A}, +{0x016E5B, 0x016E7B}, +{0x016E5C, 0x016E7C}, +{0x016E5D, 0x016E7D}, +{0x016E5E, 0x016E7E}, +{0x016E5F, 0x016E7F}, +{0x01E900, 0x01E922}, +{0x01E901, 0x01E923}, +{0x01E902, 0x01E924}, +{0x01E903, 0x01E925}, +{0x01E904, 0x01E926}, +{0x01E905, 0x01E927}, +{0x01E906, 0x01E928}, +{0x01E907, 0x01E929}, +{0x01E908, 0x01E92A}, +{0x01E909, 0x01E92B}, +{0x01E90A, 0x01E92C}, +{0x01E90B, 0x01E92D}, +{0x01E90C, 0x01E92E}, +{0x01E90D, 0x01E92F}, +{0x01E90E, 0x01E930}, +{0x01E90F, 0x01E931}, +{0x01E910, 0x01E932}, +{0x01E911, 0x01E933}, +{0x01E912, 0x01E934}, +{0x01E913, 0x01E935}, +{0x01E914, 0x01E936}, +{0x01E915, 0x01E937}, +{0x01E916, 0x01E938}, +{0x01E917, 0x01E939}, +{0x01E918, 0x01E93A}, +{0x01E919, 0x01E93B}, +{0x01E91A, 0x01E93C}, +{0x01E91B, 0x01E93D}, +{0x01E91C, 0x01E93E}, +{0x01E91D, 0x01E93F}, +{0x01E91E, 0x01E940}, +{0x01E91F, 0x01E941}, +{0x01E920, 0x01E942}, +{0x01E921, 0x01E943}, }; -const std::vector> unicode_ranges_whitespace = { -{0x00000009, 0x0000000D}, {0x00000020, 0x00000020}, {0x00000085, 0x00000085}, {0x000000A0, 0x000000A0}, -{0x00001680, 0x00001680}, {0x00002000, 0x0000200A}, {0x00002028, 0x00002029}, {0x0000202F, 0x0000202F}, -{0x0000205F, 0x0000205F}, {0x00003000, 0x00003000}, +const std::unordered_map unicode_map_uppercase = { +{0x000061, 0x000041}, +{0x000062, 0x000042}, +{0x000063, 0x000043}, +{0x000064, 0x000044}, +{0x000065, 0x000045}, +{0x000066, 0x000046}, +{0x000067, 0x000047}, +{0x000068, 0x000048}, +{0x000069, 0x000049}, +{0x00006A, 0x00004A}, +{0x00006B, 0x00004B}, +{0x00006C, 0x00004C}, +{0x00006D, 0x00004D}, +{0x00006E, 0x00004E}, +{0x00006F, 0x00004F}, +{0x000070, 0x000050}, +{0x000071, 0x000051}, +{0x000072, 0x000052}, +{0x000073, 0x000053}, +{0x000074, 0x000054}, +{0x000075, 0x000055}, +{0x000076, 0x000056}, +{0x000077, 0x000057}, +{0x000078, 0x000058}, +{0x000079, 0x000059}, +{0x00007A, 0x00005A}, +{0x0000B5, 0x00039C}, +{0x0000DF, 0x000053}, +{0x0000E0, 0x0000C0}, +{0x0000E1, 0x0000C1}, +{0x0000E2, 0x0000C2}, +{0x0000E3, 0x0000C3}, +{0x0000E4, 0x0000C4}, +{0x0000E5, 0x0000C5}, +{0x0000E6, 0x0000C6}, +{0x0000E7, 0x0000C7}, +{0x0000E8, 0x0000C8}, +{0x0000E9, 0x0000C9}, +{0x0000EA, 0x0000CA}, +{0x0000EB, 0x0000CB}, +{0x0000EC, 0x0000CC}, +{0x0000ED, 0x0000CD}, +{0x0000EE, 0x0000CE}, +{0x0000EF, 0x0000CF}, +{0x0000F0, 0x0000D0}, +{0x0000F1, 0x0000D1}, +{0x0000F2, 0x0000D2}, +{0x0000F3, 0x0000D3}, +{0x0000F4, 0x0000D4}, +{0x0000F5, 0x0000D5}, +{0x0000F6, 0x0000D6}, +{0x0000F8, 0x0000D8}, +{0x0000F9, 0x0000D9}, +{0x0000FA, 0x0000DA}, +{0x0000FB, 0x0000DB}, +{0x0000FC, 0x0000DC}, +{0x0000FD, 0x0000DD}, +{0x0000FE, 0x0000DE}, +{0x0000FF, 0x000178}, +{0x000101, 0x000100}, +{0x000103, 0x000102}, +{0x000105, 0x000104}, +{0x000107, 0x000106}, +{0x000109, 0x000108}, +{0x00010B, 0x00010A}, +{0x00010D, 0x00010C}, +{0x00010F, 0x00010E}, +{0x000111, 0x000110}, +{0x000113, 0x000112}, +{0x000115, 0x000114}, +{0x000117, 0x000116}, +{0x000119, 0x000118}, +{0x00011B, 0x00011A}, +{0x00011D, 0x00011C}, +{0x00011F, 0x00011E}, +{0x000121, 0x000120}, +{0x000123, 0x000122}, +{0x000125, 0x000124}, +{0x000127, 0x000126}, +{0x000129, 0x000128}, +{0x00012B, 0x00012A}, +{0x00012D, 0x00012C}, +{0x00012F, 0x00012E}, +{0x000131, 0x000049}, +{0x000133, 0x000132}, +{0x000135, 0x000134}, +{0x000137, 0x000136}, +{0x00013A, 0x000139}, +{0x00013C, 0x00013B}, +{0x00013E, 0x00013D}, +{0x000140, 0x00013F}, +{0x000142, 0x000141}, +{0x000144, 0x000143}, +{0x000146, 0x000145}, +{0x000148, 0x000147}, +{0x000149, 0x0002BC}, +{0x00014B, 0x00014A}, +{0x00014D, 0x00014C}, +{0x00014F, 0x00014E}, +{0x000151, 0x000150}, +{0x000153, 0x000152}, +{0x000155, 0x000154}, +{0x000157, 0x000156}, +{0x000159, 0x000158}, +{0x00015B, 0x00015A}, +{0x00015D, 0x00015C}, +{0x00015F, 0x00015E}, +{0x000161, 0x000160}, +{0x000163, 0x000162}, +{0x000165, 0x000164}, +{0x000167, 0x000166}, +{0x000169, 0x000168}, +{0x00016B, 0x00016A}, +{0x00016D, 0x00016C}, +{0x00016F, 0x00016E}, +{0x000171, 0x000170}, +{0x000173, 0x000172}, +{0x000175, 0x000174}, +{0x000177, 0x000176}, +{0x00017A, 0x000179}, +{0x00017C, 0x00017B}, +{0x00017E, 0x00017D}, +{0x00017F, 0x000053}, +{0x000180, 0x000243}, +{0x000183, 0x000182}, +{0x000185, 0x000184}, +{0x000188, 0x000187}, +{0x00018C, 0x00018B}, +{0x000192, 0x000191}, +{0x000195, 0x0001F6}, +{0x000199, 0x000198}, +{0x00019A, 0x00023D}, +{0x00019E, 0x000220}, +{0x0001A1, 0x0001A0}, +{0x0001A3, 0x0001A2}, +{0x0001A5, 0x0001A4}, +{0x0001A8, 0x0001A7}, +{0x0001AD, 0x0001AC}, +{0x0001B0, 0x0001AF}, +{0x0001B4, 0x0001B3}, +{0x0001B6, 0x0001B5}, +{0x0001B9, 0x0001B8}, +{0x0001BD, 0x0001BC}, +{0x0001BF, 0x0001F7}, +{0x0001C5, 0x0001C4}, +{0x0001C6, 0x0001C4}, +{0x0001C8, 0x0001C7}, +{0x0001C9, 0x0001C7}, +{0x0001CB, 0x0001CA}, +{0x0001CC, 0x0001CA}, +{0x0001CE, 0x0001CD}, +{0x0001D0, 0x0001CF}, +{0x0001D2, 0x0001D1}, +{0x0001D4, 0x0001D3}, +{0x0001D6, 0x0001D5}, +{0x0001D8, 0x0001D7}, +{0x0001DA, 0x0001D9}, +{0x0001DC, 0x0001DB}, +{0x0001DD, 0x00018E}, +{0x0001DF, 0x0001DE}, +{0x0001E1, 0x0001E0}, +{0x0001E3, 0x0001E2}, +{0x0001E5, 0x0001E4}, +{0x0001E7, 0x0001E6}, +{0x0001E9, 0x0001E8}, +{0x0001EB, 0x0001EA}, +{0x0001ED, 0x0001EC}, +{0x0001EF, 0x0001EE}, +{0x0001F0, 0x00004A}, +{0x0001F2, 0x0001F1}, +{0x0001F3, 0x0001F1}, +{0x0001F5, 0x0001F4}, +{0x0001F9, 0x0001F8}, +{0x0001FB, 0x0001FA}, +{0x0001FD, 0x0001FC}, +{0x0001FF, 0x0001FE}, +{0x000201, 0x000200}, +{0x000203, 0x000202}, +{0x000205, 0x000204}, +{0x000207, 0x000206}, +{0x000209, 0x000208}, +{0x00020B, 0x00020A}, +{0x00020D, 0x00020C}, +{0x00020F, 0x00020E}, +{0x000211, 0x000210}, +{0x000213, 0x000212}, +{0x000215, 0x000214}, +{0x000217, 0x000216}, +{0x000219, 0x000218}, +{0x00021B, 0x00021A}, +{0x00021D, 0x00021C}, +{0x00021F, 0x00021E}, +{0x000223, 0x000222}, +{0x000225, 0x000224}, +{0x000227, 0x000226}, +{0x000229, 0x000228}, +{0x00022B, 0x00022A}, +{0x00022D, 0x00022C}, +{0x00022F, 0x00022E}, +{0x000231, 0x000230}, +{0x000233, 0x000232}, +{0x00023C, 0x00023B}, +{0x00023F, 0x002C7E}, +{0x000240, 0x002C7F}, +{0x000242, 0x000241}, +{0x000247, 0x000246}, +{0x000249, 0x000248}, +{0x00024B, 0x00024A}, +{0x00024D, 0x00024C}, +{0x00024F, 0x00024E}, +{0x000250, 0x002C6F}, +{0x000251, 0x002C6D}, +{0x000252, 0x002C70}, +{0x000253, 0x000181}, +{0x000254, 0x000186}, +{0x000256, 0x000189}, +{0x000257, 0x00018A}, +{0x000259, 0x00018F}, +{0x00025B, 0x000190}, +{0x00025C, 0x00A7AB}, +{0x000260, 0x000193}, +{0x000261, 0x00A7AC}, +{0x000263, 0x000194}, +{0x000265, 0x00A78D}, +{0x000266, 0x00A7AA}, +{0x000268, 0x000197}, +{0x000269, 0x000196}, +{0x00026A, 0x00A7AE}, +{0x00026B, 0x002C62}, +{0x00026C, 0x00A7AD}, +{0x00026F, 0x00019C}, +{0x000271, 0x002C6E}, +{0x000272, 0x00019D}, +{0x000275, 0x00019F}, +{0x00027D, 0x002C64}, +{0x000280, 0x0001A6}, +{0x000282, 0x00A7C5}, +{0x000283, 0x0001A9}, +{0x000287, 0x00A7B1}, +{0x000288, 0x0001AE}, +{0x000289, 0x000244}, +{0x00028A, 0x0001B1}, +{0x00028B, 0x0001B2}, +{0x00028C, 0x000245}, +{0x000292, 0x0001B7}, +{0x00029D, 0x00A7B2}, +{0x00029E, 0x00A7B0}, +{0x000345, 0x000399}, +{0x000371, 0x000370}, +{0x000373, 0x000372}, +{0x000377, 0x000376}, +{0x00037B, 0x0003FD}, +{0x00037C, 0x0003FE}, +{0x00037D, 0x0003FF}, +{0x000390, 0x000399}, +{0x0003AC, 0x000386}, +{0x0003AD, 0x000388}, +{0x0003AE, 0x000389}, +{0x0003AF, 0x00038A}, +{0x0003B0, 0x0003A5}, +{0x0003B1, 0x000391}, +{0x0003B2, 0x000392}, +{0x0003B3, 0x000393}, +{0x0003B4, 0x000394}, +{0x0003B5, 0x000395}, +{0x0003B6, 0x000396}, +{0x0003B7, 0x000397}, +{0x0003B8, 0x000398}, +{0x0003B9, 0x000399}, +{0x0003BA, 0x00039A}, +{0x0003BB, 0x00039B}, +{0x0003BC, 0x00039C}, +{0x0003BD, 0x00039D}, +{0x0003BE, 0x00039E}, +{0x0003BF, 0x00039F}, +{0x0003C0, 0x0003A0}, +{0x0003C1, 0x0003A1}, +{0x0003C2, 0x0003A3}, +{0x0003C3, 0x0003A3}, +{0x0003C4, 0x0003A4}, +{0x0003C5, 0x0003A5}, +{0x0003C6, 0x0003A6}, +{0x0003C7, 0x0003A7}, +{0x0003C8, 0x0003A8}, +{0x0003C9, 0x0003A9}, +{0x0003CA, 0x0003AA}, +{0x0003CB, 0x0003AB}, +{0x0003CC, 0x00038C}, +{0x0003CD, 0x00038E}, +{0x0003CE, 0x00038F}, +{0x0003D0, 0x000392}, +{0x0003D1, 0x000398}, +{0x0003D5, 0x0003A6}, +{0x0003D6, 0x0003A0}, +{0x0003D7, 0x0003CF}, +{0x0003D9, 0x0003D8}, +{0x0003DB, 0x0003DA}, +{0x0003DD, 0x0003DC}, +{0x0003DF, 0x0003DE}, +{0x0003E1, 0x0003E0}, +{0x0003E3, 0x0003E2}, +{0x0003E5, 0x0003E4}, +{0x0003E7, 0x0003E6}, +{0x0003E9, 0x0003E8}, +{0x0003EB, 0x0003EA}, +{0x0003ED, 0x0003EC}, +{0x0003EF, 0x0003EE}, +{0x0003F0, 0x00039A}, +{0x0003F1, 0x0003A1}, +{0x0003F2, 0x0003F9}, +{0x0003F3, 0x00037F}, +{0x0003F5, 0x000395}, +{0x0003F8, 0x0003F7}, +{0x0003FB, 0x0003FA}, +{0x000430, 0x000410}, +{0x000431, 0x000411}, +{0x000432, 0x000412}, +{0x000433, 0x000413}, +{0x000434, 0x000414}, +{0x000435, 0x000415}, +{0x000436, 0x000416}, +{0x000437, 0x000417}, +{0x000438, 0x000418}, +{0x000439, 0x000419}, +{0x00043A, 0x00041A}, +{0x00043B, 0x00041B}, +{0x00043C, 0x00041C}, +{0x00043D, 0x00041D}, +{0x00043E, 0x00041E}, +{0x00043F, 0x00041F}, +{0x000440, 0x000420}, +{0x000441, 0x000421}, +{0x000442, 0x000422}, +{0x000443, 0x000423}, +{0x000444, 0x000424}, +{0x000445, 0x000425}, +{0x000446, 0x000426}, +{0x000447, 0x000427}, +{0x000448, 0x000428}, +{0x000449, 0x000429}, +{0x00044A, 0x00042A}, +{0x00044B, 0x00042B}, +{0x00044C, 0x00042C}, +{0x00044D, 0x00042D}, +{0x00044E, 0x00042E}, +{0x00044F, 0x00042F}, +{0x000450, 0x000400}, +{0x000451, 0x000401}, +{0x000452, 0x000402}, +{0x000453, 0x000403}, +{0x000454, 0x000404}, +{0x000455, 0x000405}, +{0x000456, 0x000406}, +{0x000457, 0x000407}, +{0x000458, 0x000408}, +{0x000459, 0x000409}, +{0x00045A, 0x00040A}, +{0x00045B, 0x00040B}, +{0x00045C, 0x00040C}, +{0x00045D, 0x00040D}, +{0x00045E, 0x00040E}, +{0x00045F, 0x00040F}, +{0x000461, 0x000460}, +{0x000463, 0x000462}, +{0x000465, 0x000464}, +{0x000467, 0x000466}, +{0x000469, 0x000468}, +{0x00046B, 0x00046A}, +{0x00046D, 0x00046C}, +{0x00046F, 0x00046E}, +{0x000471, 0x000470}, +{0x000473, 0x000472}, +{0x000475, 0x000474}, +{0x000477, 0x000476}, +{0x000479, 0x000478}, +{0x00047B, 0x00047A}, +{0x00047D, 0x00047C}, +{0x00047F, 0x00047E}, +{0x000481, 0x000480}, +{0x00048B, 0x00048A}, +{0x00048D, 0x00048C}, +{0x00048F, 0x00048E}, +{0x000491, 0x000490}, +{0x000493, 0x000492}, +{0x000495, 0x000494}, +{0x000497, 0x000496}, +{0x000499, 0x000498}, +{0x00049B, 0x00049A}, +{0x00049D, 0x00049C}, +{0x00049F, 0x00049E}, +{0x0004A1, 0x0004A0}, +{0x0004A3, 0x0004A2}, +{0x0004A5, 0x0004A4}, +{0x0004A7, 0x0004A6}, +{0x0004A9, 0x0004A8}, +{0x0004AB, 0x0004AA}, +{0x0004AD, 0x0004AC}, +{0x0004AF, 0x0004AE}, +{0x0004B1, 0x0004B0}, +{0x0004B3, 0x0004B2}, +{0x0004B5, 0x0004B4}, +{0x0004B7, 0x0004B6}, +{0x0004B9, 0x0004B8}, +{0x0004BB, 0x0004BA}, +{0x0004BD, 0x0004BC}, +{0x0004BF, 0x0004BE}, +{0x0004C2, 0x0004C1}, +{0x0004C4, 0x0004C3}, +{0x0004C6, 0x0004C5}, +{0x0004C8, 0x0004C7}, +{0x0004CA, 0x0004C9}, +{0x0004CC, 0x0004CB}, +{0x0004CE, 0x0004CD}, +{0x0004CF, 0x0004C0}, +{0x0004D1, 0x0004D0}, +{0x0004D3, 0x0004D2}, +{0x0004D5, 0x0004D4}, +{0x0004D7, 0x0004D6}, +{0x0004D9, 0x0004D8}, +{0x0004DB, 0x0004DA}, +{0x0004DD, 0x0004DC}, +{0x0004DF, 0x0004DE}, +{0x0004E1, 0x0004E0}, +{0x0004E3, 0x0004E2}, +{0x0004E5, 0x0004E4}, +{0x0004E7, 0x0004E6}, +{0x0004E9, 0x0004E8}, +{0x0004EB, 0x0004EA}, +{0x0004ED, 0x0004EC}, +{0x0004EF, 0x0004EE}, +{0x0004F1, 0x0004F0}, +{0x0004F3, 0x0004F2}, +{0x0004F5, 0x0004F4}, +{0x0004F7, 0x0004F6}, +{0x0004F9, 0x0004F8}, +{0x0004FB, 0x0004FA}, +{0x0004FD, 0x0004FC}, +{0x0004FF, 0x0004FE}, +{0x000501, 0x000500}, +{0x000503, 0x000502}, +{0x000505, 0x000504}, +{0x000507, 0x000506}, +{0x000509, 0x000508}, +{0x00050B, 0x00050A}, +{0x00050D, 0x00050C}, +{0x00050F, 0x00050E}, +{0x000511, 0x000510}, +{0x000513, 0x000512}, +{0x000515, 0x000514}, +{0x000517, 0x000516}, +{0x000519, 0x000518}, +{0x00051B, 0x00051A}, +{0x00051D, 0x00051C}, +{0x00051F, 0x00051E}, +{0x000521, 0x000520}, +{0x000523, 0x000522}, +{0x000525, 0x000524}, +{0x000527, 0x000526}, +{0x000529, 0x000528}, +{0x00052B, 0x00052A}, +{0x00052D, 0x00052C}, +{0x00052F, 0x00052E}, +{0x000561, 0x000531}, +{0x000562, 0x000532}, +{0x000563, 0x000533}, +{0x000564, 0x000534}, +{0x000565, 0x000535}, +{0x000566, 0x000536}, +{0x000567, 0x000537}, +{0x000568, 0x000538}, +{0x000569, 0x000539}, +{0x00056A, 0x00053A}, +{0x00056B, 0x00053B}, +{0x00056C, 0x00053C}, +{0x00056D, 0x00053D}, +{0x00056E, 0x00053E}, +{0x00056F, 0x00053F}, +{0x000570, 0x000540}, +{0x000571, 0x000541}, +{0x000572, 0x000542}, +{0x000573, 0x000543}, +{0x000574, 0x000544}, +{0x000575, 0x000545}, +{0x000576, 0x000546}, +{0x000577, 0x000547}, +{0x000578, 0x000548}, +{0x000579, 0x000549}, +{0x00057A, 0x00054A}, +{0x00057B, 0x00054B}, +{0x00057C, 0x00054C}, +{0x00057D, 0x00054D}, +{0x00057E, 0x00054E}, +{0x00057F, 0x00054F}, +{0x000580, 0x000550}, +{0x000581, 0x000551}, +{0x000582, 0x000552}, +{0x000583, 0x000553}, +{0x000584, 0x000554}, +{0x000585, 0x000555}, +{0x000586, 0x000556}, +{0x000587, 0x000535}, +{0x0010D0, 0x001C90}, +{0x0010D1, 0x001C91}, +{0x0010D2, 0x001C92}, +{0x0010D3, 0x001C93}, +{0x0010D4, 0x001C94}, +{0x0010D5, 0x001C95}, +{0x0010D6, 0x001C96}, +{0x0010D7, 0x001C97}, +{0x0010D8, 0x001C98}, +{0x0010D9, 0x001C99}, +{0x0010DA, 0x001C9A}, +{0x0010DB, 0x001C9B}, +{0x0010DC, 0x001C9C}, +{0x0010DD, 0x001C9D}, +{0x0010DE, 0x001C9E}, +{0x0010DF, 0x001C9F}, +{0x0010E0, 0x001CA0}, +{0x0010E1, 0x001CA1}, +{0x0010E2, 0x001CA2}, +{0x0010E3, 0x001CA3}, +{0x0010E4, 0x001CA4}, +{0x0010E5, 0x001CA5}, +{0x0010E6, 0x001CA6}, +{0x0010E7, 0x001CA7}, +{0x0010E8, 0x001CA8}, +{0x0010E9, 0x001CA9}, +{0x0010EA, 0x001CAA}, +{0x0010EB, 0x001CAB}, +{0x0010EC, 0x001CAC}, +{0x0010ED, 0x001CAD}, +{0x0010EE, 0x001CAE}, +{0x0010EF, 0x001CAF}, +{0x0010F0, 0x001CB0}, +{0x0010F1, 0x001CB1}, +{0x0010F2, 0x001CB2}, +{0x0010F3, 0x001CB3}, +{0x0010F4, 0x001CB4}, +{0x0010F5, 0x001CB5}, +{0x0010F6, 0x001CB6}, +{0x0010F7, 0x001CB7}, +{0x0010F8, 0x001CB8}, +{0x0010F9, 0x001CB9}, +{0x0010FA, 0x001CBA}, +{0x0010FD, 0x001CBD}, +{0x0010FE, 0x001CBE}, +{0x0010FF, 0x001CBF}, +{0x0013F8, 0x0013F0}, +{0x0013F9, 0x0013F1}, +{0x0013FA, 0x0013F2}, +{0x0013FB, 0x0013F3}, +{0x0013FC, 0x0013F4}, +{0x0013FD, 0x0013F5}, +{0x001C80, 0x000412}, +{0x001C81, 0x000414}, +{0x001C82, 0x00041E}, +{0x001C83, 0x000421}, +{0x001C84, 0x000422}, +{0x001C85, 0x000422}, +{0x001C86, 0x00042A}, +{0x001C87, 0x000462}, +{0x001C88, 0x00A64A}, +{0x001D79, 0x00A77D}, +{0x001D7D, 0x002C63}, +{0x001D8E, 0x00A7C6}, +{0x001E01, 0x001E00}, +{0x001E03, 0x001E02}, +{0x001E05, 0x001E04}, +{0x001E07, 0x001E06}, +{0x001E09, 0x001E08}, +{0x001E0B, 0x001E0A}, +{0x001E0D, 0x001E0C}, +{0x001E0F, 0x001E0E}, +{0x001E11, 0x001E10}, +{0x001E13, 0x001E12}, +{0x001E15, 0x001E14}, +{0x001E17, 0x001E16}, +{0x001E19, 0x001E18}, +{0x001E1B, 0x001E1A}, +{0x001E1D, 0x001E1C}, +{0x001E1F, 0x001E1E}, +{0x001E21, 0x001E20}, +{0x001E23, 0x001E22}, +{0x001E25, 0x001E24}, +{0x001E27, 0x001E26}, +{0x001E29, 0x001E28}, +{0x001E2B, 0x001E2A}, +{0x001E2D, 0x001E2C}, +{0x001E2F, 0x001E2E}, +{0x001E31, 0x001E30}, +{0x001E33, 0x001E32}, +{0x001E35, 0x001E34}, +{0x001E37, 0x001E36}, +{0x001E39, 0x001E38}, +{0x001E3B, 0x001E3A}, +{0x001E3D, 0x001E3C}, +{0x001E3F, 0x001E3E}, +{0x001E41, 0x001E40}, +{0x001E43, 0x001E42}, +{0x001E45, 0x001E44}, +{0x001E47, 0x001E46}, +{0x001E49, 0x001E48}, +{0x001E4B, 0x001E4A}, +{0x001E4D, 0x001E4C}, +{0x001E4F, 0x001E4E}, +{0x001E51, 0x001E50}, +{0x001E53, 0x001E52}, +{0x001E55, 0x001E54}, +{0x001E57, 0x001E56}, +{0x001E59, 0x001E58}, +{0x001E5B, 0x001E5A}, +{0x001E5D, 0x001E5C}, +{0x001E5F, 0x001E5E}, +{0x001E61, 0x001E60}, +{0x001E63, 0x001E62}, +{0x001E65, 0x001E64}, +{0x001E67, 0x001E66}, +{0x001E69, 0x001E68}, +{0x001E6B, 0x001E6A}, +{0x001E6D, 0x001E6C}, +{0x001E6F, 0x001E6E}, +{0x001E71, 0x001E70}, +{0x001E73, 0x001E72}, +{0x001E75, 0x001E74}, +{0x001E77, 0x001E76}, +{0x001E79, 0x001E78}, +{0x001E7B, 0x001E7A}, +{0x001E7D, 0x001E7C}, +{0x001E7F, 0x001E7E}, +{0x001E81, 0x001E80}, +{0x001E83, 0x001E82}, +{0x001E85, 0x001E84}, +{0x001E87, 0x001E86}, +{0x001E89, 0x001E88}, +{0x001E8B, 0x001E8A}, +{0x001E8D, 0x001E8C}, +{0x001E8F, 0x001E8E}, +{0x001E91, 0x001E90}, +{0x001E93, 0x001E92}, +{0x001E95, 0x001E94}, +{0x001E96, 0x000048}, +{0x001E97, 0x000054}, +{0x001E98, 0x000057}, +{0x001E99, 0x000059}, +{0x001E9A, 0x000041}, +{0x001E9B, 0x001E60}, +{0x001EA1, 0x001EA0}, +{0x001EA3, 0x001EA2}, +{0x001EA5, 0x001EA4}, +{0x001EA7, 0x001EA6}, +{0x001EA9, 0x001EA8}, +{0x001EAB, 0x001EAA}, +{0x001EAD, 0x001EAC}, +{0x001EAF, 0x001EAE}, +{0x001EB1, 0x001EB0}, +{0x001EB3, 0x001EB2}, +{0x001EB5, 0x001EB4}, +{0x001EB7, 0x001EB6}, +{0x001EB9, 0x001EB8}, +{0x001EBB, 0x001EBA}, +{0x001EBD, 0x001EBC}, +{0x001EBF, 0x001EBE}, +{0x001EC1, 0x001EC0}, +{0x001EC3, 0x001EC2}, +{0x001EC5, 0x001EC4}, +{0x001EC7, 0x001EC6}, +{0x001EC9, 0x001EC8}, +{0x001ECB, 0x001ECA}, +{0x001ECD, 0x001ECC}, +{0x001ECF, 0x001ECE}, +{0x001ED1, 0x001ED0}, +{0x001ED3, 0x001ED2}, +{0x001ED5, 0x001ED4}, +{0x001ED7, 0x001ED6}, +{0x001ED9, 0x001ED8}, +{0x001EDB, 0x001EDA}, +{0x001EDD, 0x001EDC}, +{0x001EDF, 0x001EDE}, +{0x001EE1, 0x001EE0}, +{0x001EE3, 0x001EE2}, +{0x001EE5, 0x001EE4}, +{0x001EE7, 0x001EE6}, +{0x001EE9, 0x001EE8}, +{0x001EEB, 0x001EEA}, +{0x001EED, 0x001EEC}, +{0x001EEF, 0x001EEE}, +{0x001EF1, 0x001EF0}, +{0x001EF3, 0x001EF2}, +{0x001EF5, 0x001EF4}, +{0x001EF7, 0x001EF6}, +{0x001EF9, 0x001EF8}, +{0x001EFB, 0x001EFA}, +{0x001EFD, 0x001EFC}, +{0x001EFF, 0x001EFE}, +{0x001F00, 0x001F08}, +{0x001F01, 0x001F09}, +{0x001F02, 0x001F0A}, +{0x001F03, 0x001F0B}, +{0x001F04, 0x001F0C}, +{0x001F05, 0x001F0D}, +{0x001F06, 0x001F0E}, +{0x001F07, 0x001F0F}, +{0x001F10, 0x001F18}, +{0x001F11, 0x001F19}, +{0x001F12, 0x001F1A}, +{0x001F13, 0x001F1B}, +{0x001F14, 0x001F1C}, +{0x001F15, 0x001F1D}, +{0x001F20, 0x001F28}, +{0x001F21, 0x001F29}, +{0x001F22, 0x001F2A}, +{0x001F23, 0x001F2B}, +{0x001F24, 0x001F2C}, +{0x001F25, 0x001F2D}, +{0x001F26, 0x001F2E}, +{0x001F27, 0x001F2F}, +{0x001F30, 0x001F38}, +{0x001F31, 0x001F39}, +{0x001F32, 0x001F3A}, +{0x001F33, 0x001F3B}, +{0x001F34, 0x001F3C}, +{0x001F35, 0x001F3D}, +{0x001F36, 0x001F3E}, +{0x001F37, 0x001F3F}, +{0x001F40, 0x001F48}, +{0x001F41, 0x001F49}, +{0x001F42, 0x001F4A}, +{0x001F43, 0x001F4B}, +{0x001F44, 0x001F4C}, +{0x001F45, 0x001F4D}, +{0x001F50, 0x0003A5}, +{0x001F51, 0x001F59}, +{0x001F52, 0x0003A5}, +{0x001F53, 0x001F5B}, +{0x001F54, 0x0003A5}, +{0x001F55, 0x001F5D}, +{0x001F56, 0x0003A5}, +{0x001F57, 0x001F5F}, +{0x001F60, 0x001F68}, +{0x001F61, 0x001F69}, +{0x001F62, 0x001F6A}, +{0x001F63, 0x001F6B}, +{0x001F64, 0x001F6C}, +{0x001F65, 0x001F6D}, +{0x001F66, 0x001F6E}, +{0x001F67, 0x001F6F}, +{0x001F70, 0x001FBA}, +{0x001F71, 0x001FBB}, +{0x001F72, 0x001FC8}, +{0x001F73, 0x001FC9}, +{0x001F74, 0x001FCA}, +{0x001F75, 0x001FCB}, +{0x001F76, 0x001FDA}, +{0x001F77, 0x001FDB}, +{0x001F78, 0x001FF8}, +{0x001F79, 0x001FF9}, +{0x001F7A, 0x001FEA}, +{0x001F7B, 0x001FEB}, +{0x001F7C, 0x001FFA}, +{0x001F7D, 0x001FFB}, +{0x001F80, 0x001F08}, +{0x001F81, 0x001F09}, +{0x001F82, 0x001F0A}, +{0x001F83, 0x001F0B}, +{0x001F84, 0x001F0C}, +{0x001F85, 0x001F0D}, +{0x001F86, 0x001F0E}, +{0x001F87, 0x001F0F}, +{0x001F88, 0x001F08}, +{0x001F89, 0x001F09}, +{0x001F8A, 0x001F0A}, +{0x001F8B, 0x001F0B}, +{0x001F8C, 0x001F0C}, +{0x001F8D, 0x001F0D}, +{0x001F8E, 0x001F0E}, +{0x001F8F, 0x001F0F}, +{0x001F90, 0x001F28}, +{0x001F91, 0x001F29}, +{0x001F92, 0x001F2A}, +{0x001F93, 0x001F2B}, +{0x001F94, 0x001F2C}, +{0x001F95, 0x001F2D}, +{0x001F96, 0x001F2E}, +{0x001F97, 0x001F2F}, +{0x001F98, 0x001F28}, +{0x001F99, 0x001F29}, +{0x001F9A, 0x001F2A}, +{0x001F9B, 0x001F2B}, +{0x001F9C, 0x001F2C}, +{0x001F9D, 0x001F2D}, +{0x001F9E, 0x001F2E}, +{0x001F9F, 0x001F2F}, +{0x001FA0, 0x001F68}, +{0x001FA1, 0x001F69}, +{0x001FA2, 0x001F6A}, +{0x001FA3, 0x001F6B}, +{0x001FA4, 0x001F6C}, +{0x001FA5, 0x001F6D}, +{0x001FA6, 0x001F6E}, +{0x001FA7, 0x001F6F}, +{0x001FA8, 0x001F68}, +{0x001FA9, 0x001F69}, +{0x001FAA, 0x001F6A}, +{0x001FAB, 0x001F6B}, +{0x001FAC, 0x001F6C}, +{0x001FAD, 0x001F6D}, +{0x001FAE, 0x001F6E}, +{0x001FAF, 0x001F6F}, +{0x001FB0, 0x001FB8}, +{0x001FB1, 0x001FB9}, +{0x001FB2, 0x001FBA}, +{0x001FB3, 0x000391}, +{0x001FB4, 0x000386}, +{0x001FB6, 0x000391}, +{0x001FB7, 0x000391}, +{0x001FBC, 0x000391}, +{0x001FBE, 0x000399}, +{0x001FC2, 0x001FCA}, +{0x001FC3, 0x000397}, +{0x001FC4, 0x000389}, +{0x001FC6, 0x000397}, +{0x001FC7, 0x000397}, +{0x001FCC, 0x000397}, +{0x001FD0, 0x001FD8}, +{0x001FD1, 0x001FD9}, +{0x001FD2, 0x000399}, +{0x001FD3, 0x000399}, +{0x001FD6, 0x000399}, +{0x001FD7, 0x000399}, +{0x001FE0, 0x001FE8}, +{0x001FE1, 0x001FE9}, +{0x001FE2, 0x0003A5}, +{0x001FE3, 0x0003A5}, +{0x001FE4, 0x0003A1}, +{0x001FE5, 0x001FEC}, +{0x001FE6, 0x0003A5}, +{0x001FE7, 0x0003A5}, +{0x001FF2, 0x001FFA}, +{0x001FF3, 0x0003A9}, +{0x001FF4, 0x00038F}, +{0x001FF6, 0x0003A9}, +{0x001FF7, 0x0003A9}, +{0x001FFC, 0x0003A9}, +{0x00214E, 0x002132}, +{0x002170, 0x002160}, +{0x002171, 0x002161}, +{0x002172, 0x002162}, +{0x002173, 0x002163}, +{0x002174, 0x002164}, +{0x002175, 0x002165}, +{0x002176, 0x002166}, +{0x002177, 0x002167}, +{0x002178, 0x002168}, +{0x002179, 0x002169}, +{0x00217A, 0x00216A}, +{0x00217B, 0x00216B}, +{0x00217C, 0x00216C}, +{0x00217D, 0x00216D}, +{0x00217E, 0x00216E}, +{0x00217F, 0x00216F}, +{0x002184, 0x002183}, +{0x0024D0, 0x0024B6}, +{0x0024D1, 0x0024B7}, +{0x0024D2, 0x0024B8}, +{0x0024D3, 0x0024B9}, +{0x0024D4, 0x0024BA}, +{0x0024D5, 0x0024BB}, +{0x0024D6, 0x0024BC}, +{0x0024D7, 0x0024BD}, +{0x0024D8, 0x0024BE}, +{0x0024D9, 0x0024BF}, +{0x0024DA, 0x0024C0}, +{0x0024DB, 0x0024C1}, +{0x0024DC, 0x0024C2}, +{0x0024DD, 0x0024C3}, +{0x0024DE, 0x0024C4}, +{0x0024DF, 0x0024C5}, +{0x0024E0, 0x0024C6}, +{0x0024E1, 0x0024C7}, +{0x0024E2, 0x0024C8}, +{0x0024E3, 0x0024C9}, +{0x0024E4, 0x0024CA}, +{0x0024E5, 0x0024CB}, +{0x0024E6, 0x0024CC}, +{0x0024E7, 0x0024CD}, +{0x0024E8, 0x0024CE}, +{0x0024E9, 0x0024CF}, +{0x002C30, 0x002C00}, +{0x002C31, 0x002C01}, +{0x002C32, 0x002C02}, +{0x002C33, 0x002C03}, +{0x002C34, 0x002C04}, +{0x002C35, 0x002C05}, +{0x002C36, 0x002C06}, +{0x002C37, 0x002C07}, +{0x002C38, 0x002C08}, +{0x002C39, 0x002C09}, +{0x002C3A, 0x002C0A}, +{0x002C3B, 0x002C0B}, +{0x002C3C, 0x002C0C}, +{0x002C3D, 0x002C0D}, +{0x002C3E, 0x002C0E}, +{0x002C3F, 0x002C0F}, +{0x002C40, 0x002C10}, +{0x002C41, 0x002C11}, +{0x002C42, 0x002C12}, +{0x002C43, 0x002C13}, +{0x002C44, 0x002C14}, +{0x002C45, 0x002C15}, +{0x002C46, 0x002C16}, +{0x002C47, 0x002C17}, +{0x002C48, 0x002C18}, +{0x002C49, 0x002C19}, +{0x002C4A, 0x002C1A}, +{0x002C4B, 0x002C1B}, +{0x002C4C, 0x002C1C}, +{0x002C4D, 0x002C1D}, +{0x002C4E, 0x002C1E}, +{0x002C4F, 0x002C1F}, +{0x002C50, 0x002C20}, +{0x002C51, 0x002C21}, +{0x002C52, 0x002C22}, +{0x002C53, 0x002C23}, +{0x002C54, 0x002C24}, +{0x002C55, 0x002C25}, +{0x002C56, 0x002C26}, +{0x002C57, 0x002C27}, +{0x002C58, 0x002C28}, +{0x002C59, 0x002C29}, +{0x002C5A, 0x002C2A}, +{0x002C5B, 0x002C2B}, +{0x002C5C, 0x002C2C}, +{0x002C5D, 0x002C2D}, +{0x002C5E, 0x002C2E}, +{0x002C61, 0x002C60}, +{0x002C65, 0x00023A}, +{0x002C66, 0x00023E}, +{0x002C68, 0x002C67}, +{0x002C6A, 0x002C69}, +{0x002C6C, 0x002C6B}, +{0x002C73, 0x002C72}, +{0x002C76, 0x002C75}, +{0x002C81, 0x002C80}, +{0x002C83, 0x002C82}, +{0x002C85, 0x002C84}, +{0x002C87, 0x002C86}, +{0x002C89, 0x002C88}, +{0x002C8B, 0x002C8A}, +{0x002C8D, 0x002C8C}, +{0x002C8F, 0x002C8E}, +{0x002C91, 0x002C90}, +{0x002C93, 0x002C92}, +{0x002C95, 0x002C94}, +{0x002C97, 0x002C96}, +{0x002C99, 0x002C98}, +{0x002C9B, 0x002C9A}, +{0x002C9D, 0x002C9C}, +{0x002C9F, 0x002C9E}, +{0x002CA1, 0x002CA0}, +{0x002CA3, 0x002CA2}, +{0x002CA5, 0x002CA4}, +{0x002CA7, 0x002CA6}, +{0x002CA9, 0x002CA8}, +{0x002CAB, 0x002CAA}, +{0x002CAD, 0x002CAC}, +{0x002CAF, 0x002CAE}, +{0x002CB1, 0x002CB0}, +{0x002CB3, 0x002CB2}, +{0x002CB5, 0x002CB4}, +{0x002CB7, 0x002CB6}, +{0x002CB9, 0x002CB8}, +{0x002CBB, 0x002CBA}, +{0x002CBD, 0x002CBC}, +{0x002CBF, 0x002CBE}, +{0x002CC1, 0x002CC0}, +{0x002CC3, 0x002CC2}, +{0x002CC5, 0x002CC4}, +{0x002CC7, 0x002CC6}, +{0x002CC9, 0x002CC8}, +{0x002CCB, 0x002CCA}, +{0x002CCD, 0x002CCC}, +{0x002CCF, 0x002CCE}, +{0x002CD1, 0x002CD0}, +{0x002CD3, 0x002CD2}, +{0x002CD5, 0x002CD4}, +{0x002CD7, 0x002CD6}, +{0x002CD9, 0x002CD8}, +{0x002CDB, 0x002CDA}, +{0x002CDD, 0x002CDC}, +{0x002CDF, 0x002CDE}, +{0x002CE1, 0x002CE0}, +{0x002CE3, 0x002CE2}, +{0x002CEC, 0x002CEB}, +{0x002CEE, 0x002CED}, +{0x002CF3, 0x002CF2}, +{0x002D00, 0x0010A0}, +{0x002D01, 0x0010A1}, +{0x002D02, 0x0010A2}, +{0x002D03, 0x0010A3}, +{0x002D04, 0x0010A4}, +{0x002D05, 0x0010A5}, +{0x002D06, 0x0010A6}, +{0x002D07, 0x0010A7}, +{0x002D08, 0x0010A8}, +{0x002D09, 0x0010A9}, +{0x002D0A, 0x0010AA}, +{0x002D0B, 0x0010AB}, +{0x002D0C, 0x0010AC}, +{0x002D0D, 0x0010AD}, +{0x002D0E, 0x0010AE}, +{0x002D0F, 0x0010AF}, +{0x002D10, 0x0010B0}, +{0x002D11, 0x0010B1}, +{0x002D12, 0x0010B2}, +{0x002D13, 0x0010B3}, +{0x002D14, 0x0010B4}, +{0x002D15, 0x0010B5}, +{0x002D16, 0x0010B6}, +{0x002D17, 0x0010B7}, +{0x002D18, 0x0010B8}, +{0x002D19, 0x0010B9}, +{0x002D1A, 0x0010BA}, +{0x002D1B, 0x0010BB}, +{0x002D1C, 0x0010BC}, +{0x002D1D, 0x0010BD}, +{0x002D1E, 0x0010BE}, +{0x002D1F, 0x0010BF}, +{0x002D20, 0x0010C0}, +{0x002D21, 0x0010C1}, +{0x002D22, 0x0010C2}, +{0x002D23, 0x0010C3}, +{0x002D24, 0x0010C4}, +{0x002D25, 0x0010C5}, +{0x002D27, 0x0010C7}, +{0x002D2D, 0x0010CD}, +{0x00A641, 0x00A640}, +{0x00A643, 0x00A642}, +{0x00A645, 0x00A644}, +{0x00A647, 0x00A646}, +{0x00A649, 0x00A648}, +{0x00A64B, 0x00A64A}, +{0x00A64D, 0x00A64C}, +{0x00A64F, 0x00A64E}, +{0x00A651, 0x00A650}, +{0x00A653, 0x00A652}, +{0x00A655, 0x00A654}, +{0x00A657, 0x00A656}, +{0x00A659, 0x00A658}, +{0x00A65B, 0x00A65A}, +{0x00A65D, 0x00A65C}, +{0x00A65F, 0x00A65E}, +{0x00A661, 0x00A660}, +{0x00A663, 0x00A662}, +{0x00A665, 0x00A664}, +{0x00A667, 0x00A666}, +{0x00A669, 0x00A668}, +{0x00A66B, 0x00A66A}, +{0x00A66D, 0x00A66C}, +{0x00A681, 0x00A680}, +{0x00A683, 0x00A682}, +{0x00A685, 0x00A684}, +{0x00A687, 0x00A686}, +{0x00A689, 0x00A688}, +{0x00A68B, 0x00A68A}, +{0x00A68D, 0x00A68C}, +{0x00A68F, 0x00A68E}, +{0x00A691, 0x00A690}, +{0x00A693, 0x00A692}, +{0x00A695, 0x00A694}, +{0x00A697, 0x00A696}, +{0x00A699, 0x00A698}, +{0x00A69B, 0x00A69A}, +{0x00A723, 0x00A722}, +{0x00A725, 0x00A724}, +{0x00A727, 0x00A726}, +{0x00A729, 0x00A728}, +{0x00A72B, 0x00A72A}, +{0x00A72D, 0x00A72C}, +{0x00A72F, 0x00A72E}, +{0x00A733, 0x00A732}, +{0x00A735, 0x00A734}, +{0x00A737, 0x00A736}, +{0x00A739, 0x00A738}, +{0x00A73B, 0x00A73A}, +{0x00A73D, 0x00A73C}, +{0x00A73F, 0x00A73E}, +{0x00A741, 0x00A740}, +{0x00A743, 0x00A742}, +{0x00A745, 0x00A744}, +{0x00A747, 0x00A746}, +{0x00A749, 0x00A748}, +{0x00A74B, 0x00A74A}, +{0x00A74D, 0x00A74C}, +{0x00A74F, 0x00A74E}, +{0x00A751, 0x00A750}, +{0x00A753, 0x00A752}, +{0x00A755, 0x00A754}, +{0x00A757, 0x00A756}, +{0x00A759, 0x00A758}, +{0x00A75B, 0x00A75A}, +{0x00A75D, 0x00A75C}, +{0x00A75F, 0x00A75E}, +{0x00A761, 0x00A760}, +{0x00A763, 0x00A762}, +{0x00A765, 0x00A764}, +{0x00A767, 0x00A766}, +{0x00A769, 0x00A768}, +{0x00A76B, 0x00A76A}, +{0x00A76D, 0x00A76C}, +{0x00A76F, 0x00A76E}, +{0x00A77A, 0x00A779}, +{0x00A77C, 0x00A77B}, +{0x00A77F, 0x00A77E}, +{0x00A781, 0x00A780}, +{0x00A783, 0x00A782}, +{0x00A785, 0x00A784}, +{0x00A787, 0x00A786}, +{0x00A78C, 0x00A78B}, +{0x00A791, 0x00A790}, +{0x00A793, 0x00A792}, +{0x00A794, 0x00A7C4}, +{0x00A797, 0x00A796}, +{0x00A799, 0x00A798}, +{0x00A79B, 0x00A79A}, +{0x00A79D, 0x00A79C}, +{0x00A79F, 0x00A79E}, +{0x00A7A1, 0x00A7A0}, +{0x00A7A3, 0x00A7A2}, +{0x00A7A5, 0x00A7A4}, +{0x00A7A7, 0x00A7A6}, +{0x00A7A9, 0x00A7A8}, +{0x00A7B5, 0x00A7B4}, +{0x00A7B7, 0x00A7B6}, +{0x00A7B9, 0x00A7B8}, +{0x00A7BB, 0x00A7BA}, +{0x00A7BD, 0x00A7BC}, +{0x00A7BF, 0x00A7BE}, +{0x00A7C3, 0x00A7C2}, +{0x00A7C8, 0x00A7C7}, +{0x00A7CA, 0x00A7C9}, +{0x00A7F6, 0x00A7F5}, +{0x00AB53, 0x00A7B3}, +{0x00AB70, 0x0013A0}, +{0x00AB71, 0x0013A1}, +{0x00AB72, 0x0013A2}, +{0x00AB73, 0x0013A3}, +{0x00AB74, 0x0013A4}, +{0x00AB75, 0x0013A5}, +{0x00AB76, 0x0013A6}, +{0x00AB77, 0x0013A7}, +{0x00AB78, 0x0013A8}, +{0x00AB79, 0x0013A9}, +{0x00AB7A, 0x0013AA}, +{0x00AB7B, 0x0013AB}, +{0x00AB7C, 0x0013AC}, +{0x00AB7D, 0x0013AD}, +{0x00AB7E, 0x0013AE}, +{0x00AB7F, 0x0013AF}, +{0x00AB80, 0x0013B0}, +{0x00AB81, 0x0013B1}, +{0x00AB82, 0x0013B2}, +{0x00AB83, 0x0013B3}, +{0x00AB84, 0x0013B4}, +{0x00AB85, 0x0013B5}, +{0x00AB86, 0x0013B6}, +{0x00AB87, 0x0013B7}, +{0x00AB88, 0x0013B8}, +{0x00AB89, 0x0013B9}, +{0x00AB8A, 0x0013BA}, +{0x00AB8B, 0x0013BB}, +{0x00AB8C, 0x0013BC}, +{0x00AB8D, 0x0013BD}, +{0x00AB8E, 0x0013BE}, +{0x00AB8F, 0x0013BF}, +{0x00AB90, 0x0013C0}, +{0x00AB91, 0x0013C1}, +{0x00AB92, 0x0013C2}, +{0x00AB93, 0x0013C3}, +{0x00AB94, 0x0013C4}, +{0x00AB95, 0x0013C5}, +{0x00AB96, 0x0013C6}, +{0x00AB97, 0x0013C7}, +{0x00AB98, 0x0013C8}, +{0x00AB99, 0x0013C9}, +{0x00AB9A, 0x0013CA}, +{0x00AB9B, 0x0013CB}, +{0x00AB9C, 0x0013CC}, +{0x00AB9D, 0x0013CD}, +{0x00AB9E, 0x0013CE}, +{0x00AB9F, 0x0013CF}, +{0x00ABA0, 0x0013D0}, +{0x00ABA1, 0x0013D1}, +{0x00ABA2, 0x0013D2}, +{0x00ABA3, 0x0013D3}, +{0x00ABA4, 0x0013D4}, +{0x00ABA5, 0x0013D5}, +{0x00ABA6, 0x0013D6}, +{0x00ABA7, 0x0013D7}, +{0x00ABA8, 0x0013D8}, +{0x00ABA9, 0x0013D9}, +{0x00ABAA, 0x0013DA}, +{0x00ABAB, 0x0013DB}, +{0x00ABAC, 0x0013DC}, +{0x00ABAD, 0x0013DD}, +{0x00ABAE, 0x0013DE}, +{0x00ABAF, 0x0013DF}, +{0x00ABB0, 0x0013E0}, +{0x00ABB1, 0x0013E1}, +{0x00ABB2, 0x0013E2}, +{0x00ABB3, 0x0013E3}, +{0x00ABB4, 0x0013E4}, +{0x00ABB5, 0x0013E5}, +{0x00ABB6, 0x0013E6}, +{0x00ABB7, 0x0013E7}, +{0x00ABB8, 0x0013E8}, +{0x00ABB9, 0x0013E9}, +{0x00ABBA, 0x0013EA}, +{0x00ABBB, 0x0013EB}, +{0x00ABBC, 0x0013EC}, +{0x00ABBD, 0x0013ED}, +{0x00ABBE, 0x0013EE}, +{0x00ABBF, 0x0013EF}, +{0x00FB00, 0x000046}, +{0x00FB01, 0x000046}, +{0x00FB02, 0x000046}, +{0x00FB03, 0x000046}, +{0x00FB04, 0x000046}, +{0x00FB05, 0x000053}, +{0x00FB06, 0x000053}, +{0x00FB13, 0x000544}, +{0x00FB14, 0x000544}, +{0x00FB15, 0x000544}, +{0x00FB16, 0x00054E}, +{0x00FB17, 0x000544}, +{0x00FF41, 0x00FF21}, +{0x00FF42, 0x00FF22}, +{0x00FF43, 0x00FF23}, +{0x00FF44, 0x00FF24}, +{0x00FF45, 0x00FF25}, +{0x00FF46, 0x00FF26}, +{0x00FF47, 0x00FF27}, +{0x00FF48, 0x00FF28}, +{0x00FF49, 0x00FF29}, +{0x00FF4A, 0x00FF2A}, +{0x00FF4B, 0x00FF2B}, +{0x00FF4C, 0x00FF2C}, +{0x00FF4D, 0x00FF2D}, +{0x00FF4E, 0x00FF2E}, +{0x00FF4F, 0x00FF2F}, +{0x00FF50, 0x00FF30}, +{0x00FF51, 0x00FF31}, +{0x00FF52, 0x00FF32}, +{0x00FF53, 0x00FF33}, +{0x00FF54, 0x00FF34}, +{0x00FF55, 0x00FF35}, +{0x00FF56, 0x00FF36}, +{0x00FF57, 0x00FF37}, +{0x00FF58, 0x00FF38}, +{0x00FF59, 0x00FF39}, +{0x00FF5A, 0x00FF3A}, +{0x010428, 0x010400}, +{0x010429, 0x010401}, +{0x01042A, 0x010402}, +{0x01042B, 0x010403}, +{0x01042C, 0x010404}, +{0x01042D, 0x010405}, +{0x01042E, 0x010406}, +{0x01042F, 0x010407}, +{0x010430, 0x010408}, +{0x010431, 0x010409}, +{0x010432, 0x01040A}, +{0x010433, 0x01040B}, +{0x010434, 0x01040C}, +{0x010435, 0x01040D}, +{0x010436, 0x01040E}, +{0x010437, 0x01040F}, +{0x010438, 0x010410}, +{0x010439, 0x010411}, +{0x01043A, 0x010412}, +{0x01043B, 0x010413}, +{0x01043C, 0x010414}, +{0x01043D, 0x010415}, +{0x01043E, 0x010416}, +{0x01043F, 0x010417}, +{0x010440, 0x010418}, +{0x010441, 0x010419}, +{0x010442, 0x01041A}, +{0x010443, 0x01041B}, +{0x010444, 0x01041C}, +{0x010445, 0x01041D}, +{0x010446, 0x01041E}, +{0x010447, 0x01041F}, +{0x010448, 0x010420}, +{0x010449, 0x010421}, +{0x01044A, 0x010422}, +{0x01044B, 0x010423}, +{0x01044C, 0x010424}, +{0x01044D, 0x010425}, +{0x01044E, 0x010426}, +{0x01044F, 0x010427}, +{0x0104D8, 0x0104B0}, +{0x0104D9, 0x0104B1}, +{0x0104DA, 0x0104B2}, +{0x0104DB, 0x0104B3}, +{0x0104DC, 0x0104B4}, +{0x0104DD, 0x0104B5}, +{0x0104DE, 0x0104B6}, +{0x0104DF, 0x0104B7}, +{0x0104E0, 0x0104B8}, +{0x0104E1, 0x0104B9}, +{0x0104E2, 0x0104BA}, +{0x0104E3, 0x0104BB}, +{0x0104E4, 0x0104BC}, +{0x0104E5, 0x0104BD}, +{0x0104E6, 0x0104BE}, +{0x0104E7, 0x0104BF}, +{0x0104E8, 0x0104C0}, +{0x0104E9, 0x0104C1}, +{0x0104EA, 0x0104C2}, +{0x0104EB, 0x0104C3}, +{0x0104EC, 0x0104C4}, +{0x0104ED, 0x0104C5}, +{0x0104EE, 0x0104C6}, +{0x0104EF, 0x0104C7}, +{0x0104F0, 0x0104C8}, +{0x0104F1, 0x0104C9}, +{0x0104F2, 0x0104CA}, +{0x0104F3, 0x0104CB}, +{0x0104F4, 0x0104CC}, +{0x0104F5, 0x0104CD}, +{0x0104F6, 0x0104CE}, +{0x0104F7, 0x0104CF}, +{0x0104F8, 0x0104D0}, +{0x0104F9, 0x0104D1}, +{0x0104FA, 0x0104D2}, +{0x0104FB, 0x0104D3}, +{0x010CC0, 0x010C80}, +{0x010CC1, 0x010C81}, +{0x010CC2, 0x010C82}, +{0x010CC3, 0x010C83}, +{0x010CC4, 0x010C84}, +{0x010CC5, 0x010C85}, +{0x010CC6, 0x010C86}, +{0x010CC7, 0x010C87}, +{0x010CC8, 0x010C88}, +{0x010CC9, 0x010C89}, +{0x010CCA, 0x010C8A}, +{0x010CCB, 0x010C8B}, +{0x010CCC, 0x010C8C}, +{0x010CCD, 0x010C8D}, +{0x010CCE, 0x010C8E}, +{0x010CCF, 0x010C8F}, +{0x010CD0, 0x010C90}, +{0x010CD1, 0x010C91}, +{0x010CD2, 0x010C92}, +{0x010CD3, 0x010C93}, +{0x010CD4, 0x010C94}, +{0x010CD5, 0x010C95}, +{0x010CD6, 0x010C96}, +{0x010CD7, 0x010C97}, +{0x010CD8, 0x010C98}, +{0x010CD9, 0x010C99}, +{0x010CDA, 0x010C9A}, +{0x010CDB, 0x010C9B}, +{0x010CDC, 0x010C9C}, +{0x010CDD, 0x010C9D}, +{0x010CDE, 0x010C9E}, +{0x010CDF, 0x010C9F}, +{0x010CE0, 0x010CA0}, +{0x010CE1, 0x010CA1}, +{0x010CE2, 0x010CA2}, +{0x010CE3, 0x010CA3}, +{0x010CE4, 0x010CA4}, +{0x010CE5, 0x010CA5}, +{0x010CE6, 0x010CA6}, +{0x010CE7, 0x010CA7}, +{0x010CE8, 0x010CA8}, +{0x010CE9, 0x010CA9}, +{0x010CEA, 0x010CAA}, +{0x010CEB, 0x010CAB}, +{0x010CEC, 0x010CAC}, +{0x010CED, 0x010CAD}, +{0x010CEE, 0x010CAE}, +{0x010CEF, 0x010CAF}, +{0x010CF0, 0x010CB0}, +{0x010CF1, 0x010CB1}, +{0x010CF2, 0x010CB2}, +{0x0118C0, 0x0118A0}, +{0x0118C1, 0x0118A1}, +{0x0118C2, 0x0118A2}, +{0x0118C3, 0x0118A3}, +{0x0118C4, 0x0118A4}, +{0x0118C5, 0x0118A5}, +{0x0118C6, 0x0118A6}, +{0x0118C7, 0x0118A7}, +{0x0118C8, 0x0118A8}, +{0x0118C9, 0x0118A9}, +{0x0118CA, 0x0118AA}, +{0x0118CB, 0x0118AB}, +{0x0118CC, 0x0118AC}, +{0x0118CD, 0x0118AD}, +{0x0118CE, 0x0118AE}, +{0x0118CF, 0x0118AF}, +{0x0118D0, 0x0118B0}, +{0x0118D1, 0x0118B1}, +{0x0118D2, 0x0118B2}, +{0x0118D3, 0x0118B3}, +{0x0118D4, 0x0118B4}, +{0x0118D5, 0x0118B5}, +{0x0118D6, 0x0118B6}, +{0x0118D7, 0x0118B7}, +{0x0118D8, 0x0118B8}, +{0x0118D9, 0x0118B9}, +{0x0118DA, 0x0118BA}, +{0x0118DB, 0x0118BB}, +{0x0118DC, 0x0118BC}, +{0x0118DD, 0x0118BD}, +{0x0118DE, 0x0118BE}, +{0x0118DF, 0x0118BF}, +{0x016E60, 0x016E40}, +{0x016E61, 0x016E41}, +{0x016E62, 0x016E42}, +{0x016E63, 0x016E43}, +{0x016E64, 0x016E44}, +{0x016E65, 0x016E45}, +{0x016E66, 0x016E46}, +{0x016E67, 0x016E47}, +{0x016E68, 0x016E48}, +{0x016E69, 0x016E49}, +{0x016E6A, 0x016E4A}, +{0x016E6B, 0x016E4B}, +{0x016E6C, 0x016E4C}, +{0x016E6D, 0x016E4D}, +{0x016E6E, 0x016E4E}, +{0x016E6F, 0x016E4F}, +{0x016E70, 0x016E50}, +{0x016E71, 0x016E51}, +{0x016E72, 0x016E52}, +{0x016E73, 0x016E53}, +{0x016E74, 0x016E54}, +{0x016E75, 0x016E55}, +{0x016E76, 0x016E56}, +{0x016E77, 0x016E57}, +{0x016E78, 0x016E58}, +{0x016E79, 0x016E59}, +{0x016E7A, 0x016E5A}, +{0x016E7B, 0x016E5B}, +{0x016E7C, 0x016E5C}, +{0x016E7D, 0x016E5D}, +{0x016E7E, 0x016E5E}, +{0x016E7F, 0x016E5F}, +{0x01E922, 0x01E900}, +{0x01E923, 0x01E901}, +{0x01E924, 0x01E902}, +{0x01E925, 0x01E903}, +{0x01E926, 0x01E904}, +{0x01E927, 0x01E905}, +{0x01E928, 0x01E906}, +{0x01E929, 0x01E907}, +{0x01E92A, 0x01E908}, +{0x01E92B, 0x01E909}, +{0x01E92C, 0x01E90A}, +{0x01E92D, 0x01E90B}, +{0x01E92E, 0x01E90C}, +{0x01E92F, 0x01E90D}, +{0x01E930, 0x01E90E}, +{0x01E931, 0x01E90F}, +{0x01E932, 0x01E910}, +{0x01E933, 0x01E911}, +{0x01E934, 0x01E912}, +{0x01E935, 0x01E913}, +{0x01E936, 0x01E914}, +{0x01E937, 0x01E915}, +{0x01E938, 0x01E916}, +{0x01E939, 0x01E917}, +{0x01E93A, 0x01E918}, +{0x01E93B, 0x01E919}, +{0x01E93C, 0x01E91A}, +{0x01E93D, 0x01E91B}, +{0x01E93E, 0x01E91C}, +{0x01E93F, 0x01E91D}, +{0x01E940, 0x01E91E}, +{0x01E941, 0x01E91F}, +{0x01E942, 0x01E920}, +{0x01E943, 0x01E921}, }; -const std::vector> unicode_ranges_accent_mark = { -{0x00000300, 0x0000036F}, {0x00000483, 0x00000489}, {0x00000591, 0x000005BD}, {0x000005BF, 0x000005BF}, -{0x000005C1, 0x000005C2}, {0x000005C4, 0x000005C5}, {0x000005C7, 0x000005C7}, {0x00000610, 0x0000061A}, -{0x0000064B, 0x0000065F}, {0x00000670, 0x00000670}, {0x000006D6, 0x000006DC}, {0x000006DF, 0x000006E4}, -{0x000006E7, 0x000006E8}, {0x000006EA, 0x000006ED}, {0x00000711, 0x00000711}, {0x00000730, 0x0000074A}, -{0x000007A6, 0x000007B0}, {0x000007EB, 0x000007F3}, {0x000007FD, 0x000007FD}, {0x00000816, 0x00000819}, -{0x0000081B, 0x00000823}, {0x00000825, 0x00000827}, {0x00000829, 0x0000082D}, {0x00000859, 0x0000085B}, -{0x00000898, 0x0000089F}, {0x000008CA, 0x000008E1}, {0x000008E3, 0x00000903}, {0x0000093A, 0x0000093C}, -{0x0000093E, 0x0000094F}, {0x00000951, 0x00000957}, {0x00000962, 0x00000963}, {0x00000981, 0x00000983}, -{0x000009BC, 0x000009BC}, {0x000009BE, 0x000009C4}, {0x000009C7, 0x000009C8}, {0x000009CB, 0x000009CD}, -{0x000009D7, 0x000009D7}, {0x000009E2, 0x000009E3}, {0x000009FE, 0x000009FE}, {0x00000A01, 0x00000A03}, -{0x00000A3C, 0x00000A3C}, {0x00000A3E, 0x00000A42}, {0x00000A47, 0x00000A48}, {0x00000A4B, 0x00000A4D}, -{0x00000A51, 0x00000A51}, {0x00000A70, 0x00000A71}, {0x00000A75, 0x00000A75}, {0x00000A81, 0x00000A83}, -{0x00000ABC, 0x00000ABC}, {0x00000ABE, 0x00000AC5}, {0x00000AC7, 0x00000AC9}, {0x00000ACB, 0x00000ACD}, -{0x00000AE2, 0x00000AE3}, {0x00000AFA, 0x00000AFF}, {0x00000B01, 0x00000B03}, {0x00000B3C, 0x00000B3C}, -{0x00000B3E, 0x00000B44}, {0x00000B47, 0x00000B48}, {0x00000B4B, 0x00000B4D}, {0x00000B55, 0x00000B57}, -{0x00000B62, 0x00000B63}, {0x00000B82, 0x00000B82}, {0x00000BBE, 0x00000BC2}, {0x00000BC6, 0x00000BC8}, -{0x00000BCA, 0x00000BCD}, {0x00000BD7, 0x00000BD7}, {0x00000C00, 0x00000C04}, {0x00000C3C, 0x00000C3C}, -{0x00000C3E, 0x00000C44}, {0x00000C46, 0x00000C48}, {0x00000C4A, 0x00000C4D}, {0x00000C55, 0x00000C56}, -{0x00000C62, 0x00000C63}, {0x00000C81, 0x00000C83}, {0x00000CBC, 0x00000CBC}, {0x00000CBE, 0x00000CC4}, -{0x00000CC6, 0x00000CC8}, {0x00000CCA, 0x00000CCD}, {0x00000CD5, 0x00000CD6}, {0x00000CE2, 0x00000CE3}, -{0x00000CF3, 0x00000CF3}, {0x00000D00, 0x00000D03}, {0x00000D3B, 0x00000D3C}, {0x00000D3E, 0x00000D44}, -{0x00000D46, 0x00000D48}, {0x00000D4A, 0x00000D4D}, {0x00000D57, 0x00000D57}, {0x00000D62, 0x00000D63}, -{0x00000D81, 0x00000D83}, {0x00000DCA, 0x00000DCA}, {0x00000DCF, 0x00000DD4}, {0x00000DD6, 0x00000DD6}, -{0x00000DD8, 0x00000DDF}, {0x00000DF2, 0x00000DF3}, {0x00000E31, 0x00000E31}, {0x00000E34, 0x00000E3A}, -{0x00000E47, 0x00000E4E}, {0x00000EB1, 0x00000EB1}, {0x00000EB4, 0x00000EBC}, {0x00000EC8, 0x00000ECE}, -{0x00000F18, 0x00000F19}, {0x00000F35, 0x00000F35}, {0x00000F37, 0x00000F37}, {0x00000F39, 0x00000F39}, -{0x00000F3E, 0x00000F3F}, {0x00000F71, 0x00000F84}, {0x00000F86, 0x00000F87}, {0x00000F8D, 0x00000F97}, -{0x00000F99, 0x00000FBC}, {0x00000FC6, 0x00000FC6}, {0x0000102B, 0x0000103E}, {0x00001056, 0x00001059}, -{0x0000105E, 0x00001060}, {0x00001062, 0x00001064}, {0x00001067, 0x0000106D}, {0x00001071, 0x00001074}, -{0x00001082, 0x0000108D}, {0x0000108F, 0x0000108F}, {0x0000109A, 0x0000109D}, {0x0000135D, 0x0000135F}, -{0x00001712, 0x00001715}, {0x00001732, 0x00001734}, {0x00001752, 0x00001753}, {0x00001772, 0x00001773}, -{0x000017B4, 0x000017D3}, {0x000017DD, 0x000017DD}, {0x0000180B, 0x0000180D}, {0x0000180F, 0x0000180F}, -{0x00001885, 0x00001886}, {0x000018A9, 0x000018A9}, {0x00001920, 0x0000192B}, {0x00001930, 0x0000193B}, -{0x00001A17, 0x00001A1B}, {0x00001A55, 0x00001A5E}, {0x00001A60, 0x00001A7C}, {0x00001A7F, 0x00001A7F}, -{0x00001AB0, 0x00001ACE}, {0x00001B00, 0x00001B04}, {0x00001B34, 0x00001B44}, {0x00001B6B, 0x00001B73}, -{0x00001B80, 0x00001B82}, {0x00001BA1, 0x00001BAD}, {0x00001BE6, 0x00001BF3}, {0x00001C24, 0x00001C37}, -{0x00001CD0, 0x00001CD2}, {0x00001CD4, 0x00001CE8}, {0x00001CED, 0x00001CED}, {0x00001CF4, 0x00001CF4}, -{0x00001CF7, 0x00001CF9}, {0x00001DC0, 0x00001DFF}, {0x000020D0, 0x000020F0}, {0x00002CEF, 0x00002CF1}, -{0x00002D7F, 0x00002D7F}, {0x00002DE0, 0x00002DFF}, {0x0000302A, 0x0000302F}, {0x00003099, 0x0000309A}, -{0x0000A66F, 0x0000A672}, {0x0000A674, 0x0000A67D}, {0x0000A69E, 0x0000A69F}, {0x0000A6F0, 0x0000A6F1}, -{0x0000A802, 0x0000A802}, {0x0000A806, 0x0000A806}, {0x0000A80B, 0x0000A80B}, {0x0000A823, 0x0000A827}, -{0x0000A82C, 0x0000A82C}, {0x0000A880, 0x0000A881}, {0x0000A8B4, 0x0000A8C5}, {0x0000A8E0, 0x0000A8F1}, -{0x0000A8FF, 0x0000A8FF}, {0x0000A926, 0x0000A92D}, {0x0000A947, 0x0000A953}, {0x0000A980, 0x0000A983}, -{0x0000A9B3, 0x0000A9C0}, {0x0000A9E5, 0x0000A9E5}, {0x0000AA29, 0x0000AA36}, {0x0000AA43, 0x0000AA43}, -{0x0000AA4C, 0x0000AA4D}, {0x0000AA7B, 0x0000AA7D}, {0x0000AAB0, 0x0000AAB0}, {0x0000AAB2, 0x0000AAB4}, -{0x0000AAB7, 0x0000AAB8}, {0x0000AABE, 0x0000AABF}, {0x0000AAC1, 0x0000AAC1}, {0x0000AAEB, 0x0000AAEF}, -{0x0000AAF5, 0x0000AAF6}, {0x0000ABE3, 0x0000ABEA}, {0x0000ABEC, 0x0000ABED}, {0x0000FB1E, 0x0000FB1E}, -{0x0000FE00, 0x0000FE0F}, {0x0000FE20, 0x0000FE2F}, {0x000101FD, 0x000101FD}, {0x000102E0, 0x000102E0}, -{0x00010376, 0x0001037A}, {0x00010A01, 0x00010A03}, {0x00010A05, 0x00010A06}, {0x00010A0C, 0x00010A0F}, -{0x00010A38, 0x00010A3A}, {0x00010A3F, 0x00010A3F}, {0x00010AE5, 0x00010AE6}, {0x00010D24, 0x00010D27}, -{0x00010EAB, 0x00010EAC}, {0x00010EFD, 0x00010EFF}, {0x00010F46, 0x00010F50}, {0x00010F82, 0x00010F85}, -{0x00011000, 0x00011002}, {0x00011038, 0x00011046}, {0x00011070, 0x00011070}, {0x00011073, 0x00011074}, -{0x0001107F, 0x00011082}, {0x000110B0, 0x000110BA}, {0x000110C2, 0x000110C2}, {0x00011100, 0x00011102}, -{0x00011127, 0x00011134}, {0x00011145, 0x00011146}, {0x00011173, 0x00011173}, {0x00011180, 0x00011182}, -{0x000111B3, 0x000111C0}, {0x000111C9, 0x000111CC}, {0x000111CE, 0x000111CF}, {0x0001122C, 0x00011237}, -{0x0001123E, 0x0001123E}, {0x00011241, 0x00011241}, {0x000112DF, 0x000112EA}, {0x00011300, 0x00011303}, -{0x0001133B, 0x0001133C}, {0x0001133E, 0x00011344}, {0x00011347, 0x00011348}, {0x0001134B, 0x0001134D}, -{0x00011357, 0x00011357}, {0x00011362, 0x00011363}, {0x00011366, 0x0001136C}, {0x00011370, 0x00011374}, -{0x00011435, 0x00011446}, {0x0001145E, 0x0001145E}, {0x000114B0, 0x000114C3}, {0x000115AF, 0x000115B5}, -{0x000115B8, 0x000115C0}, {0x000115DC, 0x000115DD}, {0x00011630, 0x00011640}, {0x000116AB, 0x000116B7}, -{0x0001171D, 0x0001172B}, {0x0001182C, 0x0001183A}, {0x00011930, 0x00011935}, {0x00011937, 0x00011938}, -{0x0001193B, 0x0001193E}, {0x00011940, 0x00011940}, {0x00011942, 0x00011943}, {0x000119D1, 0x000119D7}, -{0x000119DA, 0x000119E0}, {0x000119E4, 0x000119E4}, {0x00011A01, 0x00011A0A}, {0x00011A33, 0x00011A39}, -{0x00011A3B, 0x00011A3E}, {0x00011A47, 0x00011A47}, {0x00011A51, 0x00011A5B}, {0x00011A8A, 0x00011A99}, -{0x00011C2F, 0x00011C36}, {0x00011C38, 0x00011C3F}, {0x00011C92, 0x00011CA7}, {0x00011CA9, 0x00011CB6}, -{0x00011D31, 0x00011D36}, {0x00011D3A, 0x00011D3A}, {0x00011D3C, 0x00011D3D}, {0x00011D3F, 0x00011D45}, -{0x00011D47, 0x00011D47}, {0x00011D8A, 0x00011D8E}, {0x00011D90, 0x00011D91}, {0x00011D93, 0x00011D97}, -{0x00011EF3, 0x00011EF6}, {0x00011F00, 0x00011F01}, {0x00011F03, 0x00011F03}, {0x00011F34, 0x00011F3A}, -{0x00011F3E, 0x00011F42}, {0x00013440, 0x00013440}, {0x00013447, 0x00013455}, {0x00016AF0, 0x00016AF4}, -{0x00016B30, 0x00016B36}, {0x00016F4F, 0x00016F4F}, {0x00016F51, 0x00016F87}, {0x00016F8F, 0x00016F92}, -{0x00016FE4, 0x00016FE4}, {0x00016FF0, 0x00016FF1}, {0x0001BC9D, 0x0001BC9E}, {0x0001CF00, 0x0001CF2D}, -{0x0001CF30, 0x0001CF46}, {0x0001D165, 0x0001D169}, {0x0001D16D, 0x0001D172}, {0x0001D17B, 0x0001D182}, -{0x0001D185, 0x0001D18B}, {0x0001D1AA, 0x0001D1AD}, {0x0001D242, 0x0001D244}, {0x0001DA00, 0x0001DA36}, -{0x0001DA3B, 0x0001DA6C}, {0x0001DA75, 0x0001DA75}, {0x0001DA84, 0x0001DA84}, {0x0001DA9B, 0x0001DA9F}, -{0x0001DAA1, 0x0001DAAF}, {0x0001E000, 0x0001E006}, {0x0001E008, 0x0001E018}, {0x0001E01B, 0x0001E021}, -{0x0001E023, 0x0001E024}, {0x0001E026, 0x0001E02A}, {0x0001E08F, 0x0001E08F}, {0x0001E130, 0x0001E136}, -{0x0001E2AE, 0x0001E2AE}, {0x0001E2EC, 0x0001E2EF}, {0x0001E4EC, 0x0001E4EF}, {0x0001E8D0, 0x0001E8D6}, -{0x0001E944, 0x0001E94A}, {0x000E0100, 0x000E01EF}, +const std::vector unicode_ranges_nfd = { // start, last, nfd +{0x000000, 0x000000, 0x000000}, +{0x0000C0, 0x0000C5, 0x000041}, +{0x0000C7, 0x0000C7, 0x000043}, +{0x0000C8, 0x0000CB, 0x000045}, +{0x0000CC, 0x0000CF, 0x000049}, +{0x0000D1, 0x0000D1, 0x00004E}, +{0x0000D2, 0x0000D6, 0x00004F}, +{0x0000D9, 0x0000DC, 0x000055}, +{0x0000DD, 0x0000DD, 0x000059}, +{0x0000E0, 0x0000E5, 0x000061}, +{0x0000E7, 0x0000E7, 0x000063}, +{0x0000E8, 0x0000EB, 0x000065}, +{0x0000EC, 0x0000EF, 0x000069}, +{0x0000F1, 0x0000F1, 0x00006E}, +{0x0000F2, 0x0000F6, 0x00006F}, +{0x0000F9, 0x0000FC, 0x000075}, +{0x0000FD, 0x0000FD, 0x000079}, +{0x0000FF, 0x0000FF, 0x000079}, +{0x000100, 0x000100, 0x000041}, +{0x000101, 0x000101, 0x000061}, +{0x000102, 0x000102, 0x000041}, +{0x000103, 0x000103, 0x000061}, +{0x000104, 0x000104, 0x000041}, +{0x000105, 0x000105, 0x000061}, +{0x000106, 0x000106, 0x000043}, +{0x000107, 0x000107, 0x000063}, +{0x000108, 0x000108, 0x000043}, +{0x000109, 0x000109, 0x000063}, +{0x00010A, 0x00010A, 0x000043}, +{0x00010B, 0x00010B, 0x000063}, +{0x00010C, 0x00010C, 0x000043}, +{0x00010D, 0x00010D, 0x000063}, +{0x00010E, 0x00010E, 0x000044}, +{0x00010F, 0x00010F, 0x000064}, +{0x000112, 0x000112, 0x000045}, +{0x000113, 0x000113, 0x000065}, +{0x000114, 0x000114, 0x000045}, +{0x000115, 0x000115, 0x000065}, +{0x000116, 0x000116, 0x000045}, +{0x000117, 0x000117, 0x000065}, +{0x000118, 0x000118, 0x000045}, +{0x000119, 0x000119, 0x000065}, +{0x00011A, 0x00011A, 0x000045}, +{0x00011B, 0x00011B, 0x000065}, +{0x00011C, 0x00011C, 0x000047}, +{0x00011D, 0x00011D, 0x000067}, +{0x00011E, 0x00011E, 0x000047}, +{0x00011F, 0x00011F, 0x000067}, +{0x000120, 0x000120, 0x000047}, +{0x000121, 0x000121, 0x000067}, +{0x000122, 0x000122, 0x000047}, +{0x000123, 0x000123, 0x000067}, +{0x000124, 0x000124, 0x000048}, +{0x000125, 0x000125, 0x000068}, +{0x000128, 0x000128, 0x000049}, +{0x000129, 0x000129, 0x000069}, +{0x00012A, 0x00012A, 0x000049}, +{0x00012B, 0x00012B, 0x000069}, +{0x00012C, 0x00012C, 0x000049}, +{0x00012D, 0x00012D, 0x000069}, +{0x00012E, 0x00012E, 0x000049}, +{0x00012F, 0x00012F, 0x000069}, +{0x000130, 0x000130, 0x000049}, +{0x000134, 0x000134, 0x00004A}, +{0x000135, 0x000135, 0x00006A}, +{0x000136, 0x000136, 0x00004B}, +{0x000137, 0x000137, 0x00006B}, +{0x000139, 0x000139, 0x00004C}, +{0x00013A, 0x00013A, 0x00006C}, +{0x00013B, 0x00013B, 0x00004C}, +{0x00013C, 0x00013C, 0x00006C}, +{0x00013D, 0x00013D, 0x00004C}, +{0x00013E, 0x00013E, 0x00006C}, +{0x000143, 0x000143, 0x00004E}, +{0x000144, 0x000144, 0x00006E}, +{0x000145, 0x000145, 0x00004E}, +{0x000146, 0x000146, 0x00006E}, +{0x000147, 0x000147, 0x00004E}, +{0x000148, 0x000148, 0x00006E}, +{0x00014C, 0x00014C, 0x00004F}, +{0x00014D, 0x00014D, 0x00006F}, +{0x00014E, 0x00014E, 0x00004F}, +{0x00014F, 0x00014F, 0x00006F}, +{0x000150, 0x000150, 0x00004F}, +{0x000151, 0x000151, 0x00006F}, +{0x000154, 0x000154, 0x000052}, +{0x000155, 0x000155, 0x000072}, +{0x000156, 0x000156, 0x000052}, +{0x000157, 0x000157, 0x000072}, +{0x000158, 0x000158, 0x000052}, +{0x000159, 0x000159, 0x000072}, +{0x00015A, 0x00015A, 0x000053}, +{0x00015B, 0x00015B, 0x000073}, +{0x00015C, 0x00015C, 0x000053}, +{0x00015D, 0x00015D, 0x000073}, +{0x00015E, 0x00015E, 0x000053}, +{0x00015F, 0x00015F, 0x000073}, +{0x000160, 0x000160, 0x000053}, +{0x000161, 0x000161, 0x000073}, +{0x000162, 0x000162, 0x000054}, +{0x000163, 0x000163, 0x000074}, +{0x000164, 0x000164, 0x000054}, +{0x000165, 0x000165, 0x000074}, +{0x000168, 0x000168, 0x000055}, +{0x000169, 0x000169, 0x000075}, +{0x00016A, 0x00016A, 0x000055}, +{0x00016B, 0x00016B, 0x000075}, +{0x00016C, 0x00016C, 0x000055}, +{0x00016D, 0x00016D, 0x000075}, +{0x00016E, 0x00016E, 0x000055}, +{0x00016F, 0x00016F, 0x000075}, +{0x000170, 0x000170, 0x000055}, +{0x000171, 0x000171, 0x000075}, +{0x000172, 0x000172, 0x000055}, +{0x000173, 0x000173, 0x000075}, +{0x000174, 0x000174, 0x000057}, +{0x000175, 0x000175, 0x000077}, +{0x000176, 0x000176, 0x000059}, +{0x000177, 0x000177, 0x000079}, +{0x000178, 0x000178, 0x000059}, +{0x000179, 0x000179, 0x00005A}, +{0x00017A, 0x00017A, 0x00007A}, +{0x00017B, 0x00017B, 0x00005A}, +{0x00017C, 0x00017C, 0x00007A}, +{0x00017D, 0x00017D, 0x00005A}, +{0x00017E, 0x00017E, 0x00007A}, +{0x0001A0, 0x0001A0, 0x00004F}, +{0x0001A1, 0x0001A1, 0x00006F}, +{0x0001AF, 0x0001AF, 0x000055}, +{0x0001B0, 0x0001B0, 0x000075}, +{0x0001CD, 0x0001CD, 0x000041}, +{0x0001CE, 0x0001CE, 0x000061}, +{0x0001CF, 0x0001CF, 0x000049}, +{0x0001D0, 0x0001D0, 0x000069}, +{0x0001D1, 0x0001D1, 0x00004F}, +{0x0001D2, 0x0001D2, 0x00006F}, +{0x0001D3, 0x0001D3, 0x000055}, +{0x0001D4, 0x0001D4, 0x000075}, +{0x0001D5, 0x0001D5, 0x000055}, +{0x0001D6, 0x0001D6, 0x000075}, +{0x0001D7, 0x0001D7, 0x000055}, +{0x0001D8, 0x0001D8, 0x000075}, +{0x0001D9, 0x0001D9, 0x000055}, +{0x0001DA, 0x0001DA, 0x000075}, +{0x0001DB, 0x0001DB, 0x000055}, +{0x0001DC, 0x0001DC, 0x000075}, +{0x0001DE, 0x0001DE, 0x000041}, +{0x0001DF, 0x0001DF, 0x000061}, +{0x0001E0, 0x0001E0, 0x000041}, +{0x0001E1, 0x0001E1, 0x000061}, +{0x0001E2, 0x0001E2, 0x0000C6}, +{0x0001E3, 0x0001E3, 0x0000E6}, +{0x0001E6, 0x0001E6, 0x000047}, +{0x0001E7, 0x0001E7, 0x000067}, +{0x0001E8, 0x0001E8, 0x00004B}, +{0x0001E9, 0x0001E9, 0x00006B}, +{0x0001EA, 0x0001EA, 0x00004F}, +{0x0001EB, 0x0001EB, 0x00006F}, +{0x0001EC, 0x0001EC, 0x00004F}, +{0x0001ED, 0x0001ED, 0x00006F}, +{0x0001EE, 0x0001EE, 0x0001B7}, +{0x0001EF, 0x0001EF, 0x000292}, +{0x0001F0, 0x0001F0, 0x00006A}, +{0x0001F4, 0x0001F4, 0x000047}, +{0x0001F5, 0x0001F5, 0x000067}, +{0x0001F8, 0x0001F8, 0x00004E}, +{0x0001F9, 0x0001F9, 0x00006E}, +{0x0001FA, 0x0001FA, 0x000041}, +{0x0001FB, 0x0001FB, 0x000061}, +{0x0001FC, 0x0001FC, 0x0000C6}, +{0x0001FD, 0x0001FD, 0x0000E6}, +{0x0001FE, 0x0001FE, 0x0000D8}, +{0x0001FF, 0x0001FF, 0x0000F8}, +{0x000200, 0x000200, 0x000041}, +{0x000201, 0x000201, 0x000061}, +{0x000202, 0x000202, 0x000041}, +{0x000203, 0x000203, 0x000061}, +{0x000204, 0x000204, 0x000045}, +{0x000205, 0x000205, 0x000065}, +{0x000206, 0x000206, 0x000045}, +{0x000207, 0x000207, 0x000065}, +{0x000208, 0x000208, 0x000049}, +{0x000209, 0x000209, 0x000069}, +{0x00020A, 0x00020A, 0x000049}, +{0x00020B, 0x00020B, 0x000069}, +{0x00020C, 0x00020C, 0x00004F}, +{0x00020D, 0x00020D, 0x00006F}, +{0x00020E, 0x00020E, 0x00004F}, +{0x00020F, 0x00020F, 0x00006F}, +{0x000210, 0x000210, 0x000052}, +{0x000211, 0x000211, 0x000072}, +{0x000212, 0x000212, 0x000052}, +{0x000213, 0x000213, 0x000072}, +{0x000214, 0x000214, 0x000055}, +{0x000215, 0x000215, 0x000075}, +{0x000216, 0x000216, 0x000055}, +{0x000217, 0x000217, 0x000075}, +{0x000218, 0x000218, 0x000053}, +{0x000219, 0x000219, 0x000073}, +{0x00021A, 0x00021A, 0x000054}, +{0x00021B, 0x00021B, 0x000074}, +{0x00021E, 0x00021E, 0x000048}, +{0x00021F, 0x00021F, 0x000068}, +{0x000226, 0x000226, 0x000041}, +{0x000227, 0x000227, 0x000061}, +{0x000228, 0x000228, 0x000045}, +{0x000229, 0x000229, 0x000065}, +{0x00022A, 0x00022A, 0x00004F}, +{0x00022B, 0x00022B, 0x00006F}, +{0x00022C, 0x00022C, 0x00004F}, +{0x00022D, 0x00022D, 0x00006F}, +{0x00022E, 0x00022E, 0x00004F}, +{0x00022F, 0x00022F, 0x00006F}, +{0x000230, 0x000230, 0x00004F}, +{0x000231, 0x000231, 0x00006F}, +{0x000232, 0x000232, 0x000059}, +{0x000233, 0x000233, 0x000079}, +{0x000340, 0x000340, 0x000300}, +{0x000341, 0x000341, 0x000301}, +{0x000343, 0x000343, 0x000313}, +{0x000344, 0x000344, 0x000308}, +{0x000374, 0x000374, 0x0002B9}, +{0x00037E, 0x00037E, 0x00003B}, +{0x000385, 0x000385, 0x0000A8}, +{0x000386, 0x000386, 0x000391}, +{0x000387, 0x000387, 0x0000B7}, +{0x000388, 0x000388, 0x000395}, +{0x000389, 0x000389, 0x000397}, +{0x00038A, 0x00038A, 0x000399}, +{0x00038C, 0x00038C, 0x00039F}, +{0x00038E, 0x00038E, 0x0003A5}, +{0x00038F, 0x00038F, 0x0003A9}, +{0x000390, 0x000390, 0x0003B9}, +{0x0003AA, 0x0003AA, 0x000399}, +{0x0003AB, 0x0003AB, 0x0003A5}, +{0x0003AC, 0x0003AC, 0x0003B1}, +{0x0003AD, 0x0003AD, 0x0003B5}, +{0x0003AE, 0x0003AE, 0x0003B7}, +{0x0003AF, 0x0003AF, 0x0003B9}, +{0x0003B0, 0x0003B0, 0x0003C5}, +{0x0003CA, 0x0003CA, 0x0003B9}, +{0x0003CB, 0x0003CB, 0x0003C5}, +{0x0003CC, 0x0003CC, 0x0003BF}, +{0x0003CD, 0x0003CD, 0x0003C5}, +{0x0003CE, 0x0003CE, 0x0003C9}, +{0x0003D3, 0x0003D4, 0x0003D2}, +{0x000400, 0x000401, 0x000415}, +{0x000403, 0x000403, 0x000413}, +{0x000407, 0x000407, 0x000406}, +{0x00040C, 0x00040C, 0x00041A}, +{0x00040D, 0x00040D, 0x000418}, +{0x00040E, 0x00040E, 0x000423}, +{0x000419, 0x000419, 0x000418}, +{0x000439, 0x000439, 0x000438}, +{0x000450, 0x000451, 0x000435}, +{0x000453, 0x000453, 0x000433}, +{0x000457, 0x000457, 0x000456}, +{0x00045C, 0x00045C, 0x00043A}, +{0x00045D, 0x00045D, 0x000438}, +{0x00045E, 0x00045E, 0x000443}, +{0x000476, 0x000476, 0x000474}, +{0x000477, 0x000477, 0x000475}, +{0x0004C1, 0x0004C1, 0x000416}, +{0x0004C2, 0x0004C2, 0x000436}, +{0x0004D0, 0x0004D0, 0x000410}, +{0x0004D1, 0x0004D1, 0x000430}, +{0x0004D2, 0x0004D2, 0x000410}, +{0x0004D3, 0x0004D3, 0x000430}, +{0x0004D6, 0x0004D6, 0x000415}, +{0x0004D7, 0x0004D7, 0x000435}, +{0x0004DA, 0x0004DA, 0x0004D8}, +{0x0004DB, 0x0004DB, 0x0004D9}, +{0x0004DC, 0x0004DC, 0x000416}, +{0x0004DD, 0x0004DD, 0x000436}, +{0x0004DE, 0x0004DE, 0x000417}, +{0x0004DF, 0x0004DF, 0x000437}, +{0x0004E2, 0x0004E2, 0x000418}, +{0x0004E3, 0x0004E3, 0x000438}, +{0x0004E4, 0x0004E4, 0x000418}, +{0x0004E5, 0x0004E5, 0x000438}, +{0x0004E6, 0x0004E6, 0x00041E}, +{0x0004E7, 0x0004E7, 0x00043E}, +{0x0004EA, 0x0004EA, 0x0004E8}, +{0x0004EB, 0x0004EB, 0x0004E9}, +{0x0004EC, 0x0004EC, 0x00042D}, +{0x0004ED, 0x0004ED, 0x00044D}, +{0x0004EE, 0x0004EE, 0x000423}, +{0x0004EF, 0x0004EF, 0x000443}, +{0x0004F0, 0x0004F0, 0x000423}, +{0x0004F1, 0x0004F1, 0x000443}, +{0x0004F2, 0x0004F2, 0x000423}, +{0x0004F3, 0x0004F3, 0x000443}, +{0x0004F4, 0x0004F4, 0x000427}, +{0x0004F5, 0x0004F5, 0x000447}, +{0x0004F8, 0x0004F8, 0x00042B}, +{0x0004F9, 0x0004F9, 0x00044B}, +{0x000622, 0x000623, 0x000627}, +{0x000624, 0x000624, 0x000648}, +{0x000625, 0x000625, 0x000627}, +{0x000626, 0x000626, 0x00064A}, +{0x0006C0, 0x0006C0, 0x0006D5}, +{0x0006C2, 0x0006C2, 0x0006C1}, +{0x0006D3, 0x0006D3, 0x0006D2}, +{0x000929, 0x000929, 0x000928}, +{0x000931, 0x000931, 0x000930}, +{0x000934, 0x000934, 0x000933}, +{0x000958, 0x000958, 0x000915}, +{0x000959, 0x000959, 0x000916}, +{0x00095A, 0x00095A, 0x000917}, +{0x00095B, 0x00095B, 0x00091C}, +{0x00095C, 0x00095C, 0x000921}, +{0x00095D, 0x00095D, 0x000922}, +{0x00095E, 0x00095E, 0x00092B}, +{0x00095F, 0x00095F, 0x00092F}, +{0x0009CB, 0x0009CC, 0x0009C7}, +{0x0009DC, 0x0009DC, 0x0009A1}, +{0x0009DD, 0x0009DD, 0x0009A2}, +{0x0009DF, 0x0009DF, 0x0009AF}, +{0x000A33, 0x000A33, 0x000A32}, +{0x000A36, 0x000A36, 0x000A38}, +{0x000A59, 0x000A59, 0x000A16}, +{0x000A5A, 0x000A5A, 0x000A17}, +{0x000A5B, 0x000A5B, 0x000A1C}, +{0x000A5E, 0x000A5E, 0x000A2B}, +{0x000B48, 0x000B48, 0x000B47}, +{0x000B4B, 0x000B4C, 0x000B47}, +{0x000B5C, 0x000B5C, 0x000B21}, +{0x000B5D, 0x000B5D, 0x000B22}, +{0x000B94, 0x000B94, 0x000B92}, +{0x000BCA, 0x000BCA, 0x000BC6}, +{0x000BCB, 0x000BCB, 0x000BC7}, +{0x000BCC, 0x000BCC, 0x000BC6}, +{0x000C48, 0x000C48, 0x000C46}, +{0x000CC0, 0x000CC0, 0x000CBF}, +{0x000CC7, 0x000CC8, 0x000CC6}, +{0x000CCA, 0x000CCB, 0x000CC6}, +{0x000D4A, 0x000D4A, 0x000D46}, +{0x000D4B, 0x000D4B, 0x000D47}, +{0x000D4C, 0x000D4C, 0x000D46}, +{0x000DDA, 0x000DDA, 0x000DD9}, +{0x000DDC, 0x000DDE, 0x000DD9}, +{0x000F43, 0x000F43, 0x000F42}, +{0x000F4D, 0x000F4D, 0x000F4C}, +{0x000F52, 0x000F52, 0x000F51}, +{0x000F57, 0x000F57, 0x000F56}, +{0x000F5C, 0x000F5C, 0x000F5B}, +{0x000F69, 0x000F69, 0x000F40}, +{0x000F73, 0x000F73, 0x000F71}, +{0x000F75, 0x000F75, 0x000F71}, +{0x000F76, 0x000F76, 0x000FB2}, +{0x000F78, 0x000F78, 0x000FB3}, +{0x000F81, 0x000F81, 0x000F71}, +{0x000F93, 0x000F93, 0x000F92}, +{0x000F9D, 0x000F9D, 0x000F9C}, +{0x000FA2, 0x000FA2, 0x000FA1}, +{0x000FA7, 0x000FA7, 0x000FA6}, +{0x000FAC, 0x000FAC, 0x000FAB}, +{0x000FB9, 0x000FB9, 0x000F90}, +{0x001026, 0x001026, 0x001025}, +{0x001B06, 0x001B06, 0x001B05}, +{0x001B08, 0x001B08, 0x001B07}, +{0x001B0A, 0x001B0A, 0x001B09}, +{0x001B0C, 0x001B0C, 0x001B0B}, +{0x001B0E, 0x001B0E, 0x001B0D}, +{0x001B12, 0x001B12, 0x001B11}, +{0x001B3B, 0x001B3B, 0x001B3A}, +{0x001B3D, 0x001B3D, 0x001B3C}, +{0x001B40, 0x001B40, 0x001B3E}, +{0x001B41, 0x001B41, 0x001B3F}, +{0x001B43, 0x001B43, 0x001B42}, +{0x001E00, 0x001E00, 0x000041}, +{0x001E01, 0x001E01, 0x000061}, +{0x001E02, 0x001E02, 0x000042}, +{0x001E03, 0x001E03, 0x000062}, +{0x001E04, 0x001E04, 0x000042}, +{0x001E05, 0x001E05, 0x000062}, +{0x001E06, 0x001E06, 0x000042}, +{0x001E07, 0x001E07, 0x000062}, +{0x001E08, 0x001E08, 0x000043}, +{0x001E09, 0x001E09, 0x000063}, +{0x001E0A, 0x001E0A, 0x000044}, +{0x001E0B, 0x001E0B, 0x000064}, +{0x001E0C, 0x001E0C, 0x000044}, +{0x001E0D, 0x001E0D, 0x000064}, +{0x001E0E, 0x001E0E, 0x000044}, +{0x001E0F, 0x001E0F, 0x000064}, +{0x001E10, 0x001E10, 0x000044}, +{0x001E11, 0x001E11, 0x000064}, +{0x001E12, 0x001E12, 0x000044}, +{0x001E13, 0x001E13, 0x000064}, +{0x001E14, 0x001E14, 0x000045}, +{0x001E15, 0x001E15, 0x000065}, +{0x001E16, 0x001E16, 0x000045}, +{0x001E17, 0x001E17, 0x000065}, +{0x001E18, 0x001E18, 0x000045}, +{0x001E19, 0x001E19, 0x000065}, +{0x001E1A, 0x001E1A, 0x000045}, +{0x001E1B, 0x001E1B, 0x000065}, +{0x001E1C, 0x001E1C, 0x000045}, +{0x001E1D, 0x001E1D, 0x000065}, +{0x001E1E, 0x001E1E, 0x000046}, +{0x001E1F, 0x001E1F, 0x000066}, +{0x001E20, 0x001E20, 0x000047}, +{0x001E21, 0x001E21, 0x000067}, +{0x001E22, 0x001E22, 0x000048}, +{0x001E23, 0x001E23, 0x000068}, +{0x001E24, 0x001E24, 0x000048}, +{0x001E25, 0x001E25, 0x000068}, +{0x001E26, 0x001E26, 0x000048}, +{0x001E27, 0x001E27, 0x000068}, +{0x001E28, 0x001E28, 0x000048}, +{0x001E29, 0x001E29, 0x000068}, +{0x001E2A, 0x001E2A, 0x000048}, +{0x001E2B, 0x001E2B, 0x000068}, +{0x001E2C, 0x001E2C, 0x000049}, +{0x001E2D, 0x001E2D, 0x000069}, +{0x001E2E, 0x001E2E, 0x000049}, +{0x001E2F, 0x001E2F, 0x000069}, +{0x001E30, 0x001E30, 0x00004B}, +{0x001E31, 0x001E31, 0x00006B}, +{0x001E32, 0x001E32, 0x00004B}, +{0x001E33, 0x001E33, 0x00006B}, +{0x001E34, 0x001E34, 0x00004B}, +{0x001E35, 0x001E35, 0x00006B}, +{0x001E36, 0x001E36, 0x00004C}, +{0x001E37, 0x001E37, 0x00006C}, +{0x001E38, 0x001E38, 0x00004C}, +{0x001E39, 0x001E39, 0x00006C}, +{0x001E3A, 0x001E3A, 0x00004C}, +{0x001E3B, 0x001E3B, 0x00006C}, +{0x001E3C, 0x001E3C, 0x00004C}, +{0x001E3D, 0x001E3D, 0x00006C}, +{0x001E3E, 0x001E3E, 0x00004D}, +{0x001E3F, 0x001E3F, 0x00006D}, +{0x001E40, 0x001E40, 0x00004D}, +{0x001E41, 0x001E41, 0x00006D}, +{0x001E42, 0x001E42, 0x00004D}, +{0x001E43, 0x001E43, 0x00006D}, +{0x001E44, 0x001E44, 0x00004E}, +{0x001E45, 0x001E45, 0x00006E}, +{0x001E46, 0x001E46, 0x00004E}, +{0x001E47, 0x001E47, 0x00006E}, +{0x001E48, 0x001E48, 0x00004E}, +{0x001E49, 0x001E49, 0x00006E}, +{0x001E4A, 0x001E4A, 0x00004E}, +{0x001E4B, 0x001E4B, 0x00006E}, +{0x001E4C, 0x001E4C, 0x00004F}, +{0x001E4D, 0x001E4D, 0x00006F}, +{0x001E4E, 0x001E4E, 0x00004F}, +{0x001E4F, 0x001E4F, 0x00006F}, +{0x001E50, 0x001E50, 0x00004F}, +{0x001E51, 0x001E51, 0x00006F}, +{0x001E52, 0x001E52, 0x00004F}, +{0x001E53, 0x001E53, 0x00006F}, +{0x001E54, 0x001E54, 0x000050}, +{0x001E55, 0x001E55, 0x000070}, +{0x001E56, 0x001E56, 0x000050}, +{0x001E57, 0x001E57, 0x000070}, +{0x001E58, 0x001E58, 0x000052}, +{0x001E59, 0x001E59, 0x000072}, +{0x001E5A, 0x001E5A, 0x000052}, +{0x001E5B, 0x001E5B, 0x000072}, +{0x001E5C, 0x001E5C, 0x000052}, +{0x001E5D, 0x001E5D, 0x000072}, +{0x001E5E, 0x001E5E, 0x000052}, +{0x001E5F, 0x001E5F, 0x000072}, +{0x001E60, 0x001E60, 0x000053}, +{0x001E61, 0x001E61, 0x000073}, +{0x001E62, 0x001E62, 0x000053}, +{0x001E63, 0x001E63, 0x000073}, +{0x001E64, 0x001E64, 0x000053}, +{0x001E65, 0x001E65, 0x000073}, +{0x001E66, 0x001E66, 0x000053}, +{0x001E67, 0x001E67, 0x000073}, +{0x001E68, 0x001E68, 0x000053}, +{0x001E69, 0x001E69, 0x000073}, +{0x001E6A, 0x001E6A, 0x000054}, +{0x001E6B, 0x001E6B, 0x000074}, +{0x001E6C, 0x001E6C, 0x000054}, +{0x001E6D, 0x001E6D, 0x000074}, +{0x001E6E, 0x001E6E, 0x000054}, +{0x001E6F, 0x001E6F, 0x000074}, +{0x001E70, 0x001E70, 0x000054}, +{0x001E71, 0x001E71, 0x000074}, +{0x001E72, 0x001E72, 0x000055}, +{0x001E73, 0x001E73, 0x000075}, +{0x001E74, 0x001E74, 0x000055}, +{0x001E75, 0x001E75, 0x000075}, +{0x001E76, 0x001E76, 0x000055}, +{0x001E77, 0x001E77, 0x000075}, +{0x001E78, 0x001E78, 0x000055}, +{0x001E79, 0x001E79, 0x000075}, +{0x001E7A, 0x001E7A, 0x000055}, +{0x001E7B, 0x001E7B, 0x000075}, +{0x001E7C, 0x001E7C, 0x000056}, +{0x001E7D, 0x001E7D, 0x000076}, +{0x001E7E, 0x001E7E, 0x000056}, +{0x001E7F, 0x001E7F, 0x000076}, +{0x001E80, 0x001E80, 0x000057}, +{0x001E81, 0x001E81, 0x000077}, +{0x001E82, 0x001E82, 0x000057}, +{0x001E83, 0x001E83, 0x000077}, +{0x001E84, 0x001E84, 0x000057}, +{0x001E85, 0x001E85, 0x000077}, +{0x001E86, 0x001E86, 0x000057}, +{0x001E87, 0x001E87, 0x000077}, +{0x001E88, 0x001E88, 0x000057}, +{0x001E89, 0x001E89, 0x000077}, +{0x001E8A, 0x001E8A, 0x000058}, +{0x001E8B, 0x001E8B, 0x000078}, +{0x001E8C, 0x001E8C, 0x000058}, +{0x001E8D, 0x001E8D, 0x000078}, +{0x001E8E, 0x001E8E, 0x000059}, +{0x001E8F, 0x001E8F, 0x000079}, +{0x001E90, 0x001E90, 0x00005A}, +{0x001E91, 0x001E91, 0x00007A}, +{0x001E92, 0x001E92, 0x00005A}, +{0x001E93, 0x001E93, 0x00007A}, +{0x001E94, 0x001E94, 0x00005A}, +{0x001E95, 0x001E95, 0x00007A}, +{0x001E96, 0x001E96, 0x000068}, +{0x001E97, 0x001E97, 0x000074}, +{0x001E98, 0x001E98, 0x000077}, +{0x001E99, 0x001E99, 0x000079}, +{0x001E9B, 0x001E9B, 0x00017F}, +{0x001EA0, 0x001EA0, 0x000041}, +{0x001EA1, 0x001EA1, 0x000061}, +{0x001EA2, 0x001EA2, 0x000041}, +{0x001EA3, 0x001EA3, 0x000061}, +{0x001EA4, 0x001EA4, 0x000041}, +{0x001EA5, 0x001EA5, 0x000061}, +{0x001EA6, 0x001EA6, 0x000041}, +{0x001EA7, 0x001EA7, 0x000061}, +{0x001EA8, 0x001EA8, 0x000041}, +{0x001EA9, 0x001EA9, 0x000061}, +{0x001EAA, 0x001EAA, 0x000041}, +{0x001EAB, 0x001EAB, 0x000061}, +{0x001EAC, 0x001EAC, 0x000041}, +{0x001EAD, 0x001EAD, 0x000061}, +{0x001EAE, 0x001EAE, 0x000041}, +{0x001EAF, 0x001EAF, 0x000061}, +{0x001EB0, 0x001EB0, 0x000041}, +{0x001EB1, 0x001EB1, 0x000061}, +{0x001EB2, 0x001EB2, 0x000041}, +{0x001EB3, 0x001EB3, 0x000061}, +{0x001EB4, 0x001EB4, 0x000041}, +{0x001EB5, 0x001EB5, 0x000061}, +{0x001EB6, 0x001EB6, 0x000041}, +{0x001EB7, 0x001EB7, 0x000061}, +{0x001EB8, 0x001EB8, 0x000045}, +{0x001EB9, 0x001EB9, 0x000065}, +{0x001EBA, 0x001EBA, 0x000045}, +{0x001EBB, 0x001EBB, 0x000065}, +{0x001EBC, 0x001EBC, 0x000045}, +{0x001EBD, 0x001EBD, 0x000065}, +{0x001EBE, 0x001EBE, 0x000045}, +{0x001EBF, 0x001EBF, 0x000065}, +{0x001EC0, 0x001EC0, 0x000045}, +{0x001EC1, 0x001EC1, 0x000065}, +{0x001EC2, 0x001EC2, 0x000045}, +{0x001EC3, 0x001EC3, 0x000065}, +{0x001EC4, 0x001EC4, 0x000045}, +{0x001EC5, 0x001EC5, 0x000065}, +{0x001EC6, 0x001EC6, 0x000045}, +{0x001EC7, 0x001EC7, 0x000065}, +{0x001EC8, 0x001EC8, 0x000049}, +{0x001EC9, 0x001EC9, 0x000069}, +{0x001ECA, 0x001ECA, 0x000049}, +{0x001ECB, 0x001ECB, 0x000069}, +{0x001ECC, 0x001ECC, 0x00004F}, +{0x001ECD, 0x001ECD, 0x00006F}, +{0x001ECE, 0x001ECE, 0x00004F}, +{0x001ECF, 0x001ECF, 0x00006F}, +{0x001ED0, 0x001ED0, 0x00004F}, +{0x001ED1, 0x001ED1, 0x00006F}, +{0x001ED2, 0x001ED2, 0x00004F}, +{0x001ED3, 0x001ED3, 0x00006F}, +{0x001ED4, 0x001ED4, 0x00004F}, +{0x001ED5, 0x001ED5, 0x00006F}, +{0x001ED6, 0x001ED6, 0x00004F}, +{0x001ED7, 0x001ED7, 0x00006F}, +{0x001ED8, 0x001ED8, 0x00004F}, +{0x001ED9, 0x001ED9, 0x00006F}, +{0x001EDA, 0x001EDA, 0x00004F}, +{0x001EDB, 0x001EDB, 0x00006F}, +{0x001EDC, 0x001EDC, 0x00004F}, +{0x001EDD, 0x001EDD, 0x00006F}, +{0x001EDE, 0x001EDE, 0x00004F}, +{0x001EDF, 0x001EDF, 0x00006F}, +{0x001EE0, 0x001EE0, 0x00004F}, +{0x001EE1, 0x001EE1, 0x00006F}, +{0x001EE2, 0x001EE2, 0x00004F}, +{0x001EE3, 0x001EE3, 0x00006F}, +{0x001EE4, 0x001EE4, 0x000055}, +{0x001EE5, 0x001EE5, 0x000075}, +{0x001EE6, 0x001EE6, 0x000055}, +{0x001EE7, 0x001EE7, 0x000075}, +{0x001EE8, 0x001EE8, 0x000055}, +{0x001EE9, 0x001EE9, 0x000075}, +{0x001EEA, 0x001EEA, 0x000055}, +{0x001EEB, 0x001EEB, 0x000075}, +{0x001EEC, 0x001EEC, 0x000055}, +{0x001EED, 0x001EED, 0x000075}, +{0x001EEE, 0x001EEE, 0x000055}, +{0x001EEF, 0x001EEF, 0x000075}, +{0x001EF0, 0x001EF0, 0x000055}, +{0x001EF1, 0x001EF1, 0x000075}, +{0x001EF2, 0x001EF2, 0x000059}, +{0x001EF3, 0x001EF3, 0x000079}, +{0x001EF4, 0x001EF4, 0x000059}, +{0x001EF5, 0x001EF5, 0x000079}, +{0x001EF6, 0x001EF6, 0x000059}, +{0x001EF7, 0x001EF7, 0x000079}, +{0x001EF8, 0x001EF8, 0x000059}, +{0x001EF9, 0x001EF9, 0x000079}, +{0x001F00, 0x001F07, 0x0003B1}, +{0x001F08, 0x001F0F, 0x000391}, +{0x001F10, 0x001F15, 0x0003B5}, +{0x001F18, 0x001F1D, 0x000395}, +{0x001F20, 0x001F27, 0x0003B7}, +{0x001F28, 0x001F2F, 0x000397}, +{0x001F30, 0x001F37, 0x0003B9}, +{0x001F38, 0x001F3F, 0x000399}, +{0x001F40, 0x001F45, 0x0003BF}, +{0x001F48, 0x001F4D, 0x00039F}, +{0x001F50, 0x001F57, 0x0003C5}, +{0x001F59, 0x001F59, 0x0003A5}, +{0x001F5B, 0x001F5B, 0x0003A5}, +{0x001F5D, 0x001F5D, 0x0003A5}, +{0x001F5F, 0x001F5F, 0x0003A5}, +{0x001F60, 0x001F67, 0x0003C9}, +{0x001F68, 0x001F6F, 0x0003A9}, +{0x001F70, 0x001F71, 0x0003B1}, +{0x001F72, 0x001F73, 0x0003B5}, +{0x001F74, 0x001F75, 0x0003B7}, +{0x001F76, 0x001F77, 0x0003B9}, +{0x001F78, 0x001F79, 0x0003BF}, +{0x001F7A, 0x001F7B, 0x0003C5}, +{0x001F7C, 0x001F7D, 0x0003C9}, +{0x001F80, 0x001F87, 0x0003B1}, +{0x001F88, 0x001F8F, 0x000391}, +{0x001F90, 0x001F97, 0x0003B7}, +{0x001F98, 0x001F9F, 0x000397}, +{0x001FA0, 0x001FA7, 0x0003C9}, +{0x001FA8, 0x001FAF, 0x0003A9}, +{0x001FB0, 0x001FB4, 0x0003B1}, +{0x001FB6, 0x001FB7, 0x0003B1}, +{0x001FB8, 0x001FBC, 0x000391}, +{0x001FBE, 0x001FBE, 0x0003B9}, +{0x001FC1, 0x001FC1, 0x0000A8}, +{0x001FC2, 0x001FC4, 0x0003B7}, +{0x001FC6, 0x001FC7, 0x0003B7}, +{0x001FC8, 0x001FC9, 0x000395}, +{0x001FCA, 0x001FCC, 0x000397}, +{0x001FCD, 0x001FCF, 0x001FBF}, +{0x001FD0, 0x001FD3, 0x0003B9}, +{0x001FD6, 0x001FD7, 0x0003B9}, +{0x001FD8, 0x001FDB, 0x000399}, +{0x001FDD, 0x001FDF, 0x001FFE}, +{0x001FE0, 0x001FE3, 0x0003C5}, +{0x001FE4, 0x001FE5, 0x0003C1}, +{0x001FE6, 0x001FE7, 0x0003C5}, +{0x001FE8, 0x001FEB, 0x0003A5}, +{0x001FEC, 0x001FEC, 0x0003A1}, +{0x001FED, 0x001FEE, 0x0000A8}, +{0x001FEF, 0x001FEF, 0x000060}, +{0x001FF2, 0x001FF4, 0x0003C9}, +{0x001FF6, 0x001FF7, 0x0003C9}, +{0x001FF8, 0x001FF9, 0x00039F}, +{0x001FFA, 0x001FFC, 0x0003A9}, +{0x001FFD, 0x001FFD, 0x0000B4}, +{0x002000, 0x002000, 0x002002}, +{0x002001, 0x002001, 0x002003}, +{0x002126, 0x002126, 0x0003A9}, +{0x00212A, 0x00212A, 0x00004B}, +{0x00212B, 0x00212B, 0x000041}, +{0x00219A, 0x00219A, 0x002190}, +{0x00219B, 0x00219B, 0x002192}, +{0x0021AE, 0x0021AE, 0x002194}, +{0x0021CD, 0x0021CD, 0x0021D0}, +{0x0021CE, 0x0021CE, 0x0021D4}, +{0x0021CF, 0x0021CF, 0x0021D2}, +{0x002204, 0x002204, 0x002203}, +{0x002209, 0x002209, 0x002208}, +{0x00220C, 0x00220C, 0x00220B}, +{0x002224, 0x002224, 0x002223}, +{0x002226, 0x002226, 0x002225}, +{0x002241, 0x002241, 0x00223C}, +{0x002244, 0x002244, 0x002243}, +{0x002247, 0x002247, 0x002245}, +{0x002249, 0x002249, 0x002248}, +{0x002260, 0x002260, 0x00003D}, +{0x002262, 0x002262, 0x002261}, +{0x00226D, 0x00226D, 0x00224D}, +{0x00226E, 0x00226E, 0x00003C}, +{0x00226F, 0x00226F, 0x00003E}, +{0x002270, 0x002270, 0x002264}, +{0x002271, 0x002271, 0x002265}, +{0x002274, 0x002274, 0x002272}, +{0x002275, 0x002275, 0x002273}, +{0x002278, 0x002278, 0x002276}, +{0x002279, 0x002279, 0x002277}, +{0x002280, 0x002280, 0x00227A}, +{0x002281, 0x002281, 0x00227B}, +{0x002284, 0x002284, 0x002282}, +{0x002285, 0x002285, 0x002283}, +{0x002288, 0x002288, 0x002286}, +{0x002289, 0x002289, 0x002287}, +{0x0022AC, 0x0022AC, 0x0022A2}, +{0x0022AD, 0x0022AD, 0x0022A8}, +{0x0022AE, 0x0022AE, 0x0022A9}, +{0x0022AF, 0x0022AF, 0x0022AB}, +{0x0022E0, 0x0022E0, 0x00227C}, +{0x0022E1, 0x0022E1, 0x00227D}, +{0x0022E2, 0x0022E2, 0x002291}, +{0x0022E3, 0x0022E3, 0x002292}, +{0x0022EA, 0x0022EA, 0x0022B2}, +{0x0022EB, 0x0022EB, 0x0022B3}, +{0x0022EC, 0x0022EC, 0x0022B4}, +{0x0022ED, 0x0022ED, 0x0022B5}, +{0x002329, 0x002329, 0x003008}, +{0x00232A, 0x00232A, 0x003009}, +{0x002ADC, 0x002ADC, 0x002ADD}, +{0x00304C, 0x00304C, 0x00304B}, +{0x00304E, 0x00304E, 0x00304D}, +{0x003050, 0x003050, 0x00304F}, +{0x003052, 0x003052, 0x003051}, +{0x003054, 0x003054, 0x003053}, +{0x003056, 0x003056, 0x003055}, +{0x003058, 0x003058, 0x003057}, +{0x00305A, 0x00305A, 0x003059}, +{0x00305C, 0x00305C, 0x00305B}, +{0x00305E, 0x00305E, 0x00305D}, +{0x003060, 0x003060, 0x00305F}, +{0x003062, 0x003062, 0x003061}, +{0x003065, 0x003065, 0x003064}, +{0x003067, 0x003067, 0x003066}, +{0x003069, 0x003069, 0x003068}, +{0x003070, 0x003071, 0x00306F}, +{0x003073, 0x003074, 0x003072}, +{0x003076, 0x003077, 0x003075}, +{0x003079, 0x00307A, 0x003078}, +{0x00307C, 0x00307D, 0x00307B}, +{0x003094, 0x003094, 0x003046}, +{0x00309E, 0x00309E, 0x00309D}, +{0x0030AC, 0x0030AC, 0x0030AB}, +{0x0030AE, 0x0030AE, 0x0030AD}, +{0x0030B0, 0x0030B0, 0x0030AF}, +{0x0030B2, 0x0030B2, 0x0030B1}, +{0x0030B4, 0x0030B4, 0x0030B3}, +{0x0030B6, 0x0030B6, 0x0030B5}, +{0x0030B8, 0x0030B8, 0x0030B7}, +{0x0030BA, 0x0030BA, 0x0030B9}, +{0x0030BC, 0x0030BC, 0x0030BB}, +{0x0030BE, 0x0030BE, 0x0030BD}, +{0x0030C0, 0x0030C0, 0x0030BF}, +{0x0030C2, 0x0030C2, 0x0030C1}, +{0x0030C5, 0x0030C5, 0x0030C4}, +{0x0030C7, 0x0030C7, 0x0030C6}, +{0x0030C9, 0x0030C9, 0x0030C8}, +{0x0030D0, 0x0030D1, 0x0030CF}, +{0x0030D3, 0x0030D4, 0x0030D2}, +{0x0030D6, 0x0030D7, 0x0030D5}, +{0x0030D9, 0x0030DA, 0x0030D8}, +{0x0030DC, 0x0030DD, 0x0030DB}, +{0x0030F4, 0x0030F4, 0x0030A6}, +{0x0030F7, 0x0030F7, 0x0030EF}, +{0x0030F8, 0x0030F8, 0x0030F0}, +{0x0030F9, 0x0030F9, 0x0030F1}, +{0x0030FA, 0x0030FA, 0x0030F2}, +{0x0030FE, 0x0030FE, 0x0030FD}, +{0x00AC00, 0x00AE4B, 0x001100}, +{0x00AE4C, 0x00B097, 0x001101}, +{0x00B098, 0x00B2E3, 0x001102}, +{0x00B2E4, 0x00B52F, 0x001103}, +{0x00B530, 0x00B77B, 0x001104}, +{0x00B77C, 0x00B9C7, 0x001105}, +{0x00B9C8, 0x00BC13, 0x001106}, +{0x00BC14, 0x00BE5F, 0x001107}, +{0x00BE60, 0x00C0AB, 0x001108}, +{0x00C0AC, 0x00C2F7, 0x001109}, +{0x00C2F8, 0x00C543, 0x00110A}, +{0x00C544, 0x00C78F, 0x00110B}, +{0x00C790, 0x00C9DB, 0x00110C}, +{0x00C9DC, 0x00CC27, 0x00110D}, +{0x00CC28, 0x00CE73, 0x00110E}, +{0x00CE74, 0x00D0BF, 0x00110F}, +{0x00D0C0, 0x00D30B, 0x001110}, +{0x00D30C, 0x00D557, 0x001111}, +{0x00D558, 0x00D7A3, 0x001112}, +{0x00F900, 0x00F900, 0x008C48}, +{0x00F901, 0x00F901, 0x0066F4}, +{0x00F902, 0x00F902, 0x008ECA}, +{0x00F903, 0x00F903, 0x008CC8}, +{0x00F904, 0x00F904, 0x006ED1}, +{0x00F905, 0x00F905, 0x004E32}, +{0x00F906, 0x00F906, 0x0053E5}, +{0x00F907, 0x00F908, 0x009F9C}, +{0x00F909, 0x00F909, 0x005951}, +{0x00F90A, 0x00F90A, 0x0091D1}, +{0x00F90B, 0x00F90B, 0x005587}, +{0x00F90C, 0x00F90C, 0x005948}, +{0x00F90D, 0x00F90D, 0x0061F6}, +{0x00F90E, 0x00F90E, 0x007669}, +{0x00F90F, 0x00F90F, 0x007F85}, +{0x00F910, 0x00F910, 0x00863F}, +{0x00F911, 0x00F911, 0x0087BA}, +{0x00F912, 0x00F912, 0x0088F8}, +{0x00F913, 0x00F913, 0x00908F}, +{0x00F914, 0x00F914, 0x006A02}, +{0x00F915, 0x00F915, 0x006D1B}, +{0x00F916, 0x00F916, 0x0070D9}, +{0x00F917, 0x00F917, 0x0073DE}, +{0x00F918, 0x00F918, 0x00843D}, +{0x00F919, 0x00F919, 0x00916A}, +{0x00F91A, 0x00F91A, 0x0099F1}, +{0x00F91B, 0x00F91B, 0x004E82}, +{0x00F91C, 0x00F91C, 0x005375}, +{0x00F91D, 0x00F91D, 0x006B04}, +{0x00F91E, 0x00F91E, 0x00721B}, +{0x00F91F, 0x00F91F, 0x00862D}, +{0x00F920, 0x00F920, 0x009E1E}, +{0x00F921, 0x00F921, 0x005D50}, +{0x00F922, 0x00F922, 0x006FEB}, +{0x00F923, 0x00F923, 0x0085CD}, +{0x00F924, 0x00F924, 0x008964}, +{0x00F925, 0x00F925, 0x0062C9}, +{0x00F926, 0x00F926, 0x0081D8}, +{0x00F927, 0x00F927, 0x00881F}, +{0x00F928, 0x00F928, 0x005ECA}, +{0x00F929, 0x00F929, 0x006717}, +{0x00F92A, 0x00F92A, 0x006D6A}, +{0x00F92B, 0x00F92B, 0x0072FC}, +{0x00F92C, 0x00F92C, 0x0090CE}, +{0x00F92D, 0x00F92D, 0x004F86}, +{0x00F92E, 0x00F92E, 0x0051B7}, +{0x00F92F, 0x00F92F, 0x0052DE}, +{0x00F930, 0x00F930, 0x0064C4}, +{0x00F931, 0x00F931, 0x006AD3}, +{0x00F932, 0x00F932, 0x007210}, +{0x00F933, 0x00F933, 0x0076E7}, +{0x00F934, 0x00F934, 0x008001}, +{0x00F935, 0x00F935, 0x008606}, +{0x00F936, 0x00F936, 0x00865C}, +{0x00F937, 0x00F937, 0x008DEF}, +{0x00F938, 0x00F938, 0x009732}, +{0x00F939, 0x00F939, 0x009B6F}, +{0x00F93A, 0x00F93A, 0x009DFA}, +{0x00F93B, 0x00F93B, 0x00788C}, +{0x00F93C, 0x00F93C, 0x00797F}, +{0x00F93D, 0x00F93D, 0x007DA0}, +{0x00F93E, 0x00F93E, 0x0083C9}, +{0x00F93F, 0x00F93F, 0x009304}, +{0x00F940, 0x00F940, 0x009E7F}, +{0x00F941, 0x00F941, 0x008AD6}, +{0x00F942, 0x00F942, 0x0058DF}, +{0x00F943, 0x00F943, 0x005F04}, +{0x00F944, 0x00F944, 0x007C60}, +{0x00F945, 0x00F945, 0x00807E}, +{0x00F946, 0x00F946, 0x007262}, +{0x00F947, 0x00F947, 0x0078CA}, +{0x00F948, 0x00F948, 0x008CC2}, +{0x00F949, 0x00F949, 0x0096F7}, +{0x00F94A, 0x00F94A, 0x0058D8}, +{0x00F94B, 0x00F94B, 0x005C62}, +{0x00F94C, 0x00F94C, 0x006A13}, +{0x00F94D, 0x00F94D, 0x006DDA}, +{0x00F94E, 0x00F94E, 0x006F0F}, +{0x00F94F, 0x00F94F, 0x007D2F}, +{0x00F950, 0x00F950, 0x007E37}, +{0x00F951, 0x00F951, 0x00964B}, +{0x00F952, 0x00F952, 0x0052D2}, +{0x00F953, 0x00F953, 0x00808B}, +{0x00F954, 0x00F954, 0x0051DC}, +{0x00F955, 0x00F955, 0x0051CC}, +{0x00F956, 0x00F956, 0x007A1C}, +{0x00F957, 0x00F957, 0x007DBE}, +{0x00F958, 0x00F958, 0x0083F1}, +{0x00F959, 0x00F959, 0x009675}, +{0x00F95A, 0x00F95A, 0x008B80}, +{0x00F95B, 0x00F95B, 0x0062CF}, +{0x00F95C, 0x00F95C, 0x006A02}, +{0x00F95D, 0x00F95D, 0x008AFE}, +{0x00F95E, 0x00F95E, 0x004E39}, +{0x00F95F, 0x00F95F, 0x005BE7}, +{0x00F960, 0x00F960, 0x006012}, +{0x00F961, 0x00F961, 0x007387}, +{0x00F962, 0x00F962, 0x007570}, +{0x00F963, 0x00F963, 0x005317}, +{0x00F964, 0x00F964, 0x0078FB}, +{0x00F965, 0x00F965, 0x004FBF}, +{0x00F966, 0x00F966, 0x005FA9}, +{0x00F967, 0x00F967, 0x004E0D}, +{0x00F968, 0x00F968, 0x006CCC}, +{0x00F969, 0x00F969, 0x006578}, +{0x00F96A, 0x00F96A, 0x007D22}, +{0x00F96B, 0x00F96B, 0x0053C3}, +{0x00F96C, 0x00F96C, 0x00585E}, +{0x00F96D, 0x00F96D, 0x007701}, +{0x00F96E, 0x00F96E, 0x008449}, +{0x00F96F, 0x00F96F, 0x008AAA}, +{0x00F970, 0x00F970, 0x006BBA}, +{0x00F971, 0x00F971, 0x008FB0}, +{0x00F972, 0x00F972, 0x006C88}, +{0x00F973, 0x00F973, 0x0062FE}, +{0x00F974, 0x00F974, 0x0082E5}, +{0x00F975, 0x00F975, 0x0063A0}, +{0x00F976, 0x00F976, 0x007565}, +{0x00F977, 0x00F977, 0x004EAE}, +{0x00F978, 0x00F978, 0x005169}, +{0x00F979, 0x00F979, 0x0051C9}, +{0x00F97A, 0x00F97A, 0x006881}, +{0x00F97B, 0x00F97B, 0x007CE7}, +{0x00F97C, 0x00F97C, 0x00826F}, +{0x00F97D, 0x00F97D, 0x008AD2}, +{0x00F97E, 0x00F97E, 0x0091CF}, +{0x00F97F, 0x00F97F, 0x0052F5}, +{0x00F980, 0x00F980, 0x005442}, +{0x00F981, 0x00F981, 0x005973}, +{0x00F982, 0x00F982, 0x005EEC}, +{0x00F983, 0x00F983, 0x0065C5}, +{0x00F984, 0x00F984, 0x006FFE}, +{0x00F985, 0x00F985, 0x00792A}, +{0x00F986, 0x00F986, 0x0095AD}, +{0x00F987, 0x00F987, 0x009A6A}, +{0x00F988, 0x00F988, 0x009E97}, +{0x00F989, 0x00F989, 0x009ECE}, +{0x00F98A, 0x00F98A, 0x00529B}, +{0x00F98B, 0x00F98B, 0x0066C6}, +{0x00F98C, 0x00F98C, 0x006B77}, +{0x00F98D, 0x00F98D, 0x008F62}, +{0x00F98E, 0x00F98E, 0x005E74}, +{0x00F98F, 0x00F98F, 0x006190}, +{0x00F990, 0x00F990, 0x006200}, +{0x00F991, 0x00F991, 0x00649A}, +{0x00F992, 0x00F992, 0x006F23}, +{0x00F993, 0x00F993, 0x007149}, +{0x00F994, 0x00F994, 0x007489}, +{0x00F995, 0x00F995, 0x0079CA}, +{0x00F996, 0x00F996, 0x007DF4}, +{0x00F997, 0x00F997, 0x00806F}, +{0x00F998, 0x00F998, 0x008F26}, +{0x00F999, 0x00F999, 0x0084EE}, +{0x00F99A, 0x00F99A, 0x009023}, +{0x00F99B, 0x00F99B, 0x00934A}, +{0x00F99C, 0x00F99C, 0x005217}, +{0x00F99D, 0x00F99D, 0x0052A3}, +{0x00F99E, 0x00F99E, 0x0054BD}, +{0x00F99F, 0x00F99F, 0x0070C8}, +{0x00F9A0, 0x00F9A0, 0x0088C2}, +{0x00F9A1, 0x00F9A1, 0x008AAA}, +{0x00F9A2, 0x00F9A2, 0x005EC9}, +{0x00F9A3, 0x00F9A3, 0x005FF5}, +{0x00F9A4, 0x00F9A4, 0x00637B}, +{0x00F9A5, 0x00F9A5, 0x006BAE}, +{0x00F9A6, 0x00F9A6, 0x007C3E}, +{0x00F9A7, 0x00F9A7, 0x007375}, +{0x00F9A8, 0x00F9A8, 0x004EE4}, +{0x00F9A9, 0x00F9A9, 0x0056F9}, +{0x00F9AA, 0x00F9AA, 0x005BE7}, +{0x00F9AB, 0x00F9AB, 0x005DBA}, +{0x00F9AC, 0x00F9AC, 0x00601C}, +{0x00F9AD, 0x00F9AD, 0x0073B2}, +{0x00F9AE, 0x00F9AE, 0x007469}, +{0x00F9AF, 0x00F9AF, 0x007F9A}, +{0x00F9B0, 0x00F9B0, 0x008046}, +{0x00F9B1, 0x00F9B1, 0x009234}, +{0x00F9B2, 0x00F9B2, 0x0096F6}, +{0x00F9B3, 0x00F9B3, 0x009748}, +{0x00F9B4, 0x00F9B4, 0x009818}, +{0x00F9B5, 0x00F9B5, 0x004F8B}, +{0x00F9B6, 0x00F9B6, 0x0079AE}, +{0x00F9B7, 0x00F9B7, 0x0091B4}, +{0x00F9B8, 0x00F9B8, 0x0096B8}, +{0x00F9B9, 0x00F9B9, 0x0060E1}, +{0x00F9BA, 0x00F9BA, 0x004E86}, +{0x00F9BB, 0x00F9BB, 0x0050DA}, +{0x00F9BC, 0x00F9BC, 0x005BEE}, +{0x00F9BD, 0x00F9BD, 0x005C3F}, +{0x00F9BE, 0x00F9BE, 0x006599}, +{0x00F9BF, 0x00F9BF, 0x006A02}, +{0x00F9C0, 0x00F9C0, 0x0071CE}, +{0x00F9C1, 0x00F9C1, 0x007642}, +{0x00F9C2, 0x00F9C2, 0x0084FC}, +{0x00F9C3, 0x00F9C3, 0x00907C}, +{0x00F9C4, 0x00F9C4, 0x009F8D}, +{0x00F9C5, 0x00F9C5, 0x006688}, +{0x00F9C6, 0x00F9C6, 0x00962E}, +{0x00F9C7, 0x00F9C7, 0x005289}, +{0x00F9C8, 0x00F9C8, 0x00677B}, +{0x00F9C9, 0x00F9C9, 0x0067F3}, +{0x00F9CA, 0x00F9CA, 0x006D41}, +{0x00F9CB, 0x00F9CB, 0x006E9C}, +{0x00F9CC, 0x00F9CC, 0x007409}, +{0x00F9CD, 0x00F9CD, 0x007559}, +{0x00F9CE, 0x00F9CE, 0x00786B}, +{0x00F9CF, 0x00F9CF, 0x007D10}, +{0x00F9D0, 0x00F9D0, 0x00985E}, +{0x00F9D1, 0x00F9D1, 0x00516D}, +{0x00F9D2, 0x00F9D2, 0x00622E}, +{0x00F9D3, 0x00F9D3, 0x009678}, +{0x00F9D4, 0x00F9D4, 0x00502B}, +{0x00F9D5, 0x00F9D5, 0x005D19}, +{0x00F9D6, 0x00F9D6, 0x006DEA}, +{0x00F9D7, 0x00F9D7, 0x008F2A}, +{0x00F9D8, 0x00F9D8, 0x005F8B}, +{0x00F9D9, 0x00F9D9, 0x006144}, +{0x00F9DA, 0x00F9DA, 0x006817}, +{0x00F9DB, 0x00F9DB, 0x007387}, +{0x00F9DC, 0x00F9DC, 0x009686}, +{0x00F9DD, 0x00F9DD, 0x005229}, +{0x00F9DE, 0x00F9DE, 0x00540F}, +{0x00F9DF, 0x00F9DF, 0x005C65}, +{0x00F9E0, 0x00F9E0, 0x006613}, +{0x00F9E1, 0x00F9E1, 0x00674E}, +{0x00F9E2, 0x00F9E2, 0x0068A8}, +{0x00F9E3, 0x00F9E3, 0x006CE5}, +{0x00F9E4, 0x00F9E4, 0x007406}, +{0x00F9E5, 0x00F9E5, 0x0075E2}, +{0x00F9E6, 0x00F9E6, 0x007F79}, +{0x00F9E7, 0x00F9E7, 0x0088CF}, +{0x00F9E8, 0x00F9E8, 0x0088E1}, +{0x00F9E9, 0x00F9E9, 0x0091CC}, +{0x00F9EA, 0x00F9EA, 0x0096E2}, +{0x00F9EB, 0x00F9EB, 0x00533F}, +{0x00F9EC, 0x00F9EC, 0x006EBA}, +{0x00F9ED, 0x00F9ED, 0x00541D}, +{0x00F9EE, 0x00F9EE, 0x0071D0}, +{0x00F9EF, 0x00F9EF, 0x007498}, +{0x00F9F0, 0x00F9F0, 0x0085FA}, +{0x00F9F1, 0x00F9F1, 0x0096A3}, +{0x00F9F2, 0x00F9F2, 0x009C57}, +{0x00F9F3, 0x00F9F3, 0x009E9F}, +{0x00F9F4, 0x00F9F4, 0x006797}, +{0x00F9F5, 0x00F9F5, 0x006DCB}, +{0x00F9F6, 0x00F9F6, 0x0081E8}, +{0x00F9F7, 0x00F9F7, 0x007ACB}, +{0x00F9F8, 0x00F9F8, 0x007B20}, +{0x00F9F9, 0x00F9F9, 0x007C92}, +{0x00F9FA, 0x00F9FA, 0x0072C0}, +{0x00F9FB, 0x00F9FB, 0x007099}, +{0x00F9FC, 0x00F9FC, 0x008B58}, +{0x00F9FD, 0x00F9FD, 0x004EC0}, +{0x00F9FE, 0x00F9FE, 0x008336}, +{0x00F9FF, 0x00F9FF, 0x00523A}, +{0x00FA00, 0x00FA00, 0x005207}, +{0x00FA01, 0x00FA01, 0x005EA6}, +{0x00FA02, 0x00FA02, 0x0062D3}, +{0x00FA03, 0x00FA03, 0x007CD6}, +{0x00FA04, 0x00FA04, 0x005B85}, +{0x00FA05, 0x00FA05, 0x006D1E}, +{0x00FA06, 0x00FA06, 0x0066B4}, +{0x00FA07, 0x00FA07, 0x008F3B}, +{0x00FA08, 0x00FA08, 0x00884C}, +{0x00FA09, 0x00FA09, 0x00964D}, +{0x00FA0A, 0x00FA0A, 0x00898B}, +{0x00FA0B, 0x00FA0B, 0x005ED3}, +{0x00FA0C, 0x00FA0C, 0x005140}, +{0x00FA0D, 0x00FA0D, 0x0055C0}, +{0x00FA10, 0x00FA10, 0x00585A}, +{0x00FA12, 0x00FA12, 0x006674}, +{0x00FA15, 0x00FA15, 0x0051DE}, +{0x00FA16, 0x00FA16, 0x00732A}, +{0x00FA17, 0x00FA17, 0x0076CA}, +{0x00FA18, 0x00FA18, 0x00793C}, +{0x00FA19, 0x00FA19, 0x00795E}, +{0x00FA1A, 0x00FA1A, 0x007965}, +{0x00FA1B, 0x00FA1B, 0x00798F}, +{0x00FA1C, 0x00FA1C, 0x009756}, +{0x00FA1D, 0x00FA1D, 0x007CBE}, +{0x00FA1E, 0x00FA1E, 0x007FBD}, +{0x00FA20, 0x00FA20, 0x008612}, +{0x00FA22, 0x00FA22, 0x008AF8}, +{0x00FA25, 0x00FA25, 0x009038}, +{0x00FA26, 0x00FA26, 0x0090FD}, +{0x00FA2A, 0x00FA2A, 0x0098EF}, +{0x00FA2B, 0x00FA2B, 0x0098FC}, +{0x00FA2C, 0x00FA2C, 0x009928}, +{0x00FA2D, 0x00FA2D, 0x009DB4}, +{0x00FA2E, 0x00FA2E, 0x0090DE}, +{0x00FA2F, 0x00FA2F, 0x0096B7}, +{0x00FA30, 0x00FA30, 0x004FAE}, +{0x00FA31, 0x00FA31, 0x0050E7}, +{0x00FA32, 0x00FA32, 0x00514D}, +{0x00FA33, 0x00FA33, 0x0052C9}, +{0x00FA34, 0x00FA34, 0x0052E4}, +{0x00FA35, 0x00FA35, 0x005351}, +{0x00FA36, 0x00FA36, 0x00559D}, +{0x00FA37, 0x00FA37, 0x005606}, +{0x00FA38, 0x00FA38, 0x005668}, +{0x00FA39, 0x00FA39, 0x005840}, +{0x00FA3A, 0x00FA3A, 0x0058A8}, +{0x00FA3B, 0x00FA3B, 0x005C64}, +{0x00FA3C, 0x00FA3C, 0x005C6E}, +{0x00FA3D, 0x00FA3D, 0x006094}, +{0x00FA3E, 0x00FA3E, 0x006168}, +{0x00FA3F, 0x00FA3F, 0x00618E}, +{0x00FA40, 0x00FA40, 0x0061F2}, +{0x00FA41, 0x00FA41, 0x00654F}, +{0x00FA42, 0x00FA42, 0x0065E2}, +{0x00FA43, 0x00FA43, 0x006691}, +{0x00FA44, 0x00FA44, 0x006885}, +{0x00FA45, 0x00FA45, 0x006D77}, +{0x00FA46, 0x00FA46, 0x006E1A}, +{0x00FA47, 0x00FA47, 0x006F22}, +{0x00FA48, 0x00FA48, 0x00716E}, +{0x00FA49, 0x00FA49, 0x00722B}, +{0x00FA4A, 0x00FA4A, 0x007422}, +{0x00FA4B, 0x00FA4B, 0x007891}, +{0x00FA4C, 0x00FA4C, 0x00793E}, +{0x00FA4D, 0x00FA4D, 0x007949}, +{0x00FA4E, 0x00FA4E, 0x007948}, +{0x00FA4F, 0x00FA4F, 0x007950}, +{0x00FA50, 0x00FA50, 0x007956}, +{0x00FA51, 0x00FA51, 0x00795D}, +{0x00FA52, 0x00FA52, 0x00798D}, +{0x00FA53, 0x00FA53, 0x00798E}, +{0x00FA54, 0x00FA54, 0x007A40}, +{0x00FA55, 0x00FA55, 0x007A81}, +{0x00FA56, 0x00FA56, 0x007BC0}, +{0x00FA57, 0x00FA57, 0x007DF4}, +{0x00FA58, 0x00FA58, 0x007E09}, +{0x00FA59, 0x00FA59, 0x007E41}, +{0x00FA5A, 0x00FA5A, 0x007F72}, +{0x00FA5B, 0x00FA5B, 0x008005}, +{0x00FA5C, 0x00FA5C, 0x0081ED}, +{0x00FA5D, 0x00FA5E, 0x008279}, +{0x00FA5F, 0x00FA5F, 0x008457}, +{0x00FA60, 0x00FA60, 0x008910}, +{0x00FA61, 0x00FA61, 0x008996}, +{0x00FA62, 0x00FA62, 0x008B01}, +{0x00FA63, 0x00FA63, 0x008B39}, +{0x00FA64, 0x00FA64, 0x008CD3}, +{0x00FA65, 0x00FA65, 0x008D08}, +{0x00FA66, 0x00FA66, 0x008FB6}, +{0x00FA67, 0x00FA67, 0x009038}, +{0x00FA68, 0x00FA68, 0x0096E3}, +{0x00FA69, 0x00FA69, 0x0097FF}, +{0x00FA6A, 0x00FA6A, 0x00983B}, +{0x00FA6B, 0x00FA6B, 0x006075}, +{0x00FA6C, 0x00FA6C, 0x0242EE}, +{0x00FA6D, 0x00FA6D, 0x008218}, +{0x00FA70, 0x00FA70, 0x004E26}, +{0x00FA71, 0x00FA71, 0x0051B5}, +{0x00FA72, 0x00FA72, 0x005168}, +{0x00FA73, 0x00FA73, 0x004F80}, +{0x00FA74, 0x00FA74, 0x005145}, +{0x00FA75, 0x00FA75, 0x005180}, +{0x00FA76, 0x00FA76, 0x0052C7}, +{0x00FA77, 0x00FA77, 0x0052FA}, +{0x00FA78, 0x00FA78, 0x00559D}, +{0x00FA79, 0x00FA79, 0x005555}, +{0x00FA7A, 0x00FA7A, 0x005599}, +{0x00FA7B, 0x00FA7B, 0x0055E2}, +{0x00FA7C, 0x00FA7C, 0x00585A}, +{0x00FA7D, 0x00FA7D, 0x0058B3}, +{0x00FA7E, 0x00FA7E, 0x005944}, +{0x00FA7F, 0x00FA7F, 0x005954}, +{0x00FA80, 0x00FA80, 0x005A62}, +{0x00FA81, 0x00FA81, 0x005B28}, +{0x00FA82, 0x00FA82, 0x005ED2}, +{0x00FA83, 0x00FA83, 0x005ED9}, +{0x00FA84, 0x00FA84, 0x005F69}, +{0x00FA85, 0x00FA85, 0x005FAD}, +{0x00FA86, 0x00FA86, 0x0060D8}, +{0x00FA87, 0x00FA87, 0x00614E}, +{0x00FA88, 0x00FA88, 0x006108}, +{0x00FA89, 0x00FA89, 0x00618E}, +{0x00FA8A, 0x00FA8A, 0x006160}, +{0x00FA8B, 0x00FA8B, 0x0061F2}, +{0x00FA8C, 0x00FA8C, 0x006234}, +{0x00FA8D, 0x00FA8D, 0x0063C4}, +{0x00FA8E, 0x00FA8E, 0x00641C}, +{0x00FA8F, 0x00FA8F, 0x006452}, +{0x00FA90, 0x00FA90, 0x006556}, +{0x00FA91, 0x00FA91, 0x006674}, +{0x00FA92, 0x00FA92, 0x006717}, +{0x00FA93, 0x00FA93, 0x00671B}, +{0x00FA94, 0x00FA94, 0x006756}, +{0x00FA95, 0x00FA95, 0x006B79}, +{0x00FA96, 0x00FA96, 0x006BBA}, +{0x00FA97, 0x00FA97, 0x006D41}, +{0x00FA98, 0x00FA98, 0x006EDB}, +{0x00FA99, 0x00FA99, 0x006ECB}, +{0x00FA9A, 0x00FA9A, 0x006F22}, +{0x00FA9B, 0x00FA9B, 0x00701E}, +{0x00FA9C, 0x00FA9C, 0x00716E}, +{0x00FA9D, 0x00FA9D, 0x0077A7}, +{0x00FA9E, 0x00FA9E, 0x007235}, +{0x00FA9F, 0x00FA9F, 0x0072AF}, +{0x00FAA0, 0x00FAA0, 0x00732A}, +{0x00FAA1, 0x00FAA1, 0x007471}, +{0x00FAA2, 0x00FAA2, 0x007506}, +{0x00FAA3, 0x00FAA3, 0x00753B}, +{0x00FAA4, 0x00FAA4, 0x00761D}, +{0x00FAA5, 0x00FAA5, 0x00761F}, +{0x00FAA6, 0x00FAA6, 0x0076CA}, +{0x00FAA7, 0x00FAA7, 0x0076DB}, +{0x00FAA8, 0x00FAA8, 0x0076F4}, +{0x00FAA9, 0x00FAA9, 0x00774A}, +{0x00FAAA, 0x00FAAA, 0x007740}, +{0x00FAAB, 0x00FAAB, 0x0078CC}, +{0x00FAAC, 0x00FAAC, 0x007AB1}, +{0x00FAAD, 0x00FAAD, 0x007BC0}, +{0x00FAAE, 0x00FAAE, 0x007C7B}, +{0x00FAAF, 0x00FAAF, 0x007D5B}, +{0x00FAB0, 0x00FAB0, 0x007DF4}, +{0x00FAB1, 0x00FAB1, 0x007F3E}, +{0x00FAB2, 0x00FAB2, 0x008005}, +{0x00FAB3, 0x00FAB3, 0x008352}, +{0x00FAB4, 0x00FAB4, 0x0083EF}, +{0x00FAB5, 0x00FAB5, 0x008779}, +{0x00FAB6, 0x00FAB6, 0x008941}, +{0x00FAB7, 0x00FAB7, 0x008986}, +{0x00FAB8, 0x00FAB8, 0x008996}, +{0x00FAB9, 0x00FAB9, 0x008ABF}, +{0x00FABA, 0x00FABA, 0x008AF8}, +{0x00FABB, 0x00FABB, 0x008ACB}, +{0x00FABC, 0x00FABC, 0x008B01}, +{0x00FABD, 0x00FABD, 0x008AFE}, +{0x00FABE, 0x00FABE, 0x008AED}, +{0x00FABF, 0x00FABF, 0x008B39}, +{0x00FAC0, 0x00FAC0, 0x008B8A}, +{0x00FAC1, 0x00FAC1, 0x008D08}, +{0x00FAC2, 0x00FAC2, 0x008F38}, +{0x00FAC3, 0x00FAC3, 0x009072}, +{0x00FAC4, 0x00FAC4, 0x009199}, +{0x00FAC5, 0x00FAC5, 0x009276}, +{0x00FAC6, 0x00FAC6, 0x00967C}, +{0x00FAC7, 0x00FAC7, 0x0096E3}, +{0x00FAC8, 0x00FAC8, 0x009756}, +{0x00FAC9, 0x00FAC9, 0x0097DB}, +{0x00FACA, 0x00FACA, 0x0097FF}, +{0x00FACB, 0x00FACB, 0x00980B}, +{0x00FACC, 0x00FACC, 0x00983B}, +{0x00FACD, 0x00FACD, 0x009B12}, +{0x00FACE, 0x00FACE, 0x009F9C}, +{0x00FACF, 0x00FACF, 0x02284A}, +{0x00FAD0, 0x00FAD0, 0x022844}, +{0x00FAD1, 0x00FAD1, 0x0233D5}, +{0x00FAD2, 0x00FAD2, 0x003B9D}, +{0x00FAD3, 0x00FAD3, 0x004018}, +{0x00FAD4, 0x00FAD4, 0x004039}, +{0x00FAD5, 0x00FAD5, 0x025249}, +{0x00FAD6, 0x00FAD6, 0x025CD0}, +{0x00FAD7, 0x00FAD7, 0x027ED3}, +{0x00FAD8, 0x00FAD8, 0x009F43}, +{0x00FAD9, 0x00FAD9, 0x009F8E}, +{0x00FB1D, 0x00FB1D, 0x0005D9}, +{0x00FB1F, 0x00FB1F, 0x0005F2}, +{0x00FB2A, 0x00FB2D, 0x0005E9}, +{0x00FB2E, 0x00FB30, 0x0005D0}, +{0x00FB31, 0x00FB31, 0x0005D1}, +{0x00FB32, 0x00FB32, 0x0005D2}, +{0x00FB33, 0x00FB33, 0x0005D3}, +{0x00FB34, 0x00FB34, 0x0005D4}, +{0x00FB35, 0x00FB35, 0x0005D5}, +{0x00FB36, 0x00FB36, 0x0005D6}, +{0x00FB38, 0x00FB38, 0x0005D8}, +{0x00FB39, 0x00FB39, 0x0005D9}, +{0x00FB3A, 0x00FB3A, 0x0005DA}, +{0x00FB3B, 0x00FB3B, 0x0005DB}, +{0x00FB3C, 0x00FB3C, 0x0005DC}, +{0x00FB3E, 0x00FB3E, 0x0005DE}, +{0x00FB40, 0x00FB40, 0x0005E0}, +{0x00FB41, 0x00FB41, 0x0005E1}, +{0x00FB43, 0x00FB43, 0x0005E3}, +{0x00FB44, 0x00FB44, 0x0005E4}, +{0x00FB46, 0x00FB46, 0x0005E6}, +{0x00FB47, 0x00FB47, 0x0005E7}, +{0x00FB48, 0x00FB48, 0x0005E8}, +{0x00FB49, 0x00FB49, 0x0005E9}, +{0x00FB4A, 0x00FB4A, 0x0005EA}, +{0x00FB4B, 0x00FB4B, 0x0005D5}, +{0x00FB4C, 0x00FB4C, 0x0005D1}, +{0x00FB4D, 0x00FB4D, 0x0005DB}, +{0x00FB4E, 0x00FB4E, 0x0005E4}, +{0x01109A, 0x01109A, 0x011099}, +{0x01109C, 0x01109C, 0x01109B}, +{0x0110AB, 0x0110AB, 0x0110A5}, +{0x01112E, 0x01112E, 0x011131}, +{0x01112F, 0x01112F, 0x011132}, +{0x01134B, 0x01134C, 0x011347}, +{0x0114BB, 0x0114BC, 0x0114B9}, +{0x0114BE, 0x0114BE, 0x0114B9}, +{0x0115BA, 0x0115BA, 0x0115B8}, +{0x0115BB, 0x0115BB, 0x0115B9}, +{0x011938, 0x011938, 0x011935}, +{0x01D15E, 0x01D15E, 0x01D157}, +{0x01D15F, 0x01D164, 0x01D158}, +{0x01D1BB, 0x01D1BB, 0x01D1B9}, +{0x01D1BC, 0x01D1BC, 0x01D1BA}, +{0x01D1BD, 0x01D1BD, 0x01D1B9}, +{0x01D1BE, 0x01D1BE, 0x01D1BA}, +{0x01D1BF, 0x01D1BF, 0x01D1B9}, +{0x01D1C0, 0x01D1C0, 0x01D1BA}, +{0x02F800, 0x02F800, 0x004E3D}, +{0x02F801, 0x02F801, 0x004E38}, +{0x02F802, 0x02F802, 0x004E41}, +{0x02F803, 0x02F803, 0x020122}, +{0x02F804, 0x02F804, 0x004F60}, +{0x02F805, 0x02F805, 0x004FAE}, +{0x02F806, 0x02F806, 0x004FBB}, +{0x02F807, 0x02F807, 0x005002}, +{0x02F808, 0x02F808, 0x00507A}, +{0x02F809, 0x02F809, 0x005099}, +{0x02F80A, 0x02F80A, 0x0050E7}, +{0x02F80B, 0x02F80B, 0x0050CF}, +{0x02F80C, 0x02F80C, 0x00349E}, +{0x02F80D, 0x02F80D, 0x02063A}, +{0x02F80E, 0x02F80E, 0x00514D}, +{0x02F80F, 0x02F80F, 0x005154}, +{0x02F810, 0x02F810, 0x005164}, +{0x02F811, 0x02F811, 0x005177}, +{0x02F812, 0x02F812, 0x02051C}, +{0x02F813, 0x02F813, 0x0034B9}, +{0x02F814, 0x02F814, 0x005167}, +{0x02F815, 0x02F815, 0x00518D}, +{0x02F816, 0x02F816, 0x02054B}, +{0x02F817, 0x02F817, 0x005197}, +{0x02F818, 0x02F818, 0x0051A4}, +{0x02F819, 0x02F819, 0x004ECC}, +{0x02F81A, 0x02F81A, 0x0051AC}, +{0x02F81B, 0x02F81B, 0x0051B5}, +{0x02F81C, 0x02F81C, 0x0291DF}, +{0x02F81D, 0x02F81D, 0x0051F5}, +{0x02F81E, 0x02F81E, 0x005203}, +{0x02F81F, 0x02F81F, 0x0034DF}, +{0x02F820, 0x02F820, 0x00523B}, +{0x02F821, 0x02F821, 0x005246}, +{0x02F822, 0x02F822, 0x005272}, +{0x02F823, 0x02F823, 0x005277}, +{0x02F824, 0x02F824, 0x003515}, +{0x02F825, 0x02F825, 0x0052C7}, +{0x02F826, 0x02F826, 0x0052C9}, +{0x02F827, 0x02F827, 0x0052E4}, +{0x02F828, 0x02F828, 0x0052FA}, +{0x02F829, 0x02F829, 0x005305}, +{0x02F82A, 0x02F82A, 0x005306}, +{0x02F82B, 0x02F82B, 0x005317}, +{0x02F82C, 0x02F82C, 0x005349}, +{0x02F82D, 0x02F82D, 0x005351}, +{0x02F82E, 0x02F82E, 0x00535A}, +{0x02F82F, 0x02F82F, 0x005373}, +{0x02F830, 0x02F830, 0x00537D}, +{0x02F831, 0x02F833, 0x00537F}, +{0x02F834, 0x02F834, 0x020A2C}, +{0x02F835, 0x02F835, 0x007070}, +{0x02F836, 0x02F836, 0x0053CA}, +{0x02F837, 0x02F837, 0x0053DF}, +{0x02F838, 0x02F838, 0x020B63}, +{0x02F839, 0x02F839, 0x0053EB}, +{0x02F83A, 0x02F83A, 0x0053F1}, +{0x02F83B, 0x02F83B, 0x005406}, +{0x02F83C, 0x02F83C, 0x00549E}, +{0x02F83D, 0x02F83D, 0x005438}, +{0x02F83E, 0x02F83E, 0x005448}, +{0x02F83F, 0x02F83F, 0x005468}, +{0x02F840, 0x02F840, 0x0054A2}, +{0x02F841, 0x02F841, 0x0054F6}, +{0x02F842, 0x02F842, 0x005510}, +{0x02F843, 0x02F843, 0x005553}, +{0x02F844, 0x02F844, 0x005563}, +{0x02F845, 0x02F846, 0x005584}, +{0x02F847, 0x02F847, 0x005599}, +{0x02F848, 0x02F848, 0x0055AB}, +{0x02F849, 0x02F849, 0x0055B3}, +{0x02F84A, 0x02F84A, 0x0055C2}, +{0x02F84B, 0x02F84B, 0x005716}, +{0x02F84C, 0x02F84C, 0x005606}, +{0x02F84D, 0x02F84D, 0x005717}, +{0x02F84E, 0x02F84E, 0x005651}, +{0x02F84F, 0x02F84F, 0x005674}, +{0x02F850, 0x02F850, 0x005207}, +{0x02F851, 0x02F851, 0x0058EE}, +{0x02F852, 0x02F852, 0x0057CE}, +{0x02F853, 0x02F853, 0x0057F4}, +{0x02F854, 0x02F854, 0x00580D}, +{0x02F855, 0x02F855, 0x00578B}, +{0x02F856, 0x02F856, 0x005832}, +{0x02F857, 0x02F857, 0x005831}, +{0x02F858, 0x02F858, 0x0058AC}, +{0x02F859, 0x02F859, 0x0214E4}, +{0x02F85A, 0x02F85A, 0x0058F2}, +{0x02F85B, 0x02F85B, 0x0058F7}, +{0x02F85C, 0x02F85C, 0x005906}, +{0x02F85D, 0x02F85D, 0x00591A}, +{0x02F85E, 0x02F85E, 0x005922}, +{0x02F85F, 0x02F85F, 0x005962}, +{0x02F860, 0x02F860, 0x0216A8}, +{0x02F861, 0x02F861, 0x0216EA}, +{0x02F862, 0x02F862, 0x0059EC}, +{0x02F863, 0x02F863, 0x005A1B}, +{0x02F864, 0x02F864, 0x005A27}, +{0x02F865, 0x02F865, 0x0059D8}, +{0x02F866, 0x02F866, 0x005A66}, +{0x02F867, 0x02F867, 0x0036EE}, +{0x02F868, 0x02F868, 0x0036FC}, +{0x02F869, 0x02F869, 0x005B08}, +{0x02F86A, 0x02F86B, 0x005B3E}, +{0x02F86C, 0x02F86C, 0x0219C8}, +{0x02F86D, 0x02F86D, 0x005BC3}, +{0x02F86E, 0x02F86E, 0x005BD8}, +{0x02F86F, 0x02F86F, 0x005BE7}, +{0x02F870, 0x02F870, 0x005BF3}, +{0x02F871, 0x02F871, 0x021B18}, +{0x02F872, 0x02F872, 0x005BFF}, +{0x02F873, 0x02F873, 0x005C06}, +{0x02F874, 0x02F874, 0x005F53}, +{0x02F875, 0x02F875, 0x005C22}, +{0x02F876, 0x02F876, 0x003781}, +{0x02F877, 0x02F877, 0x005C60}, +{0x02F878, 0x02F878, 0x005C6E}, +{0x02F879, 0x02F879, 0x005CC0}, +{0x02F87A, 0x02F87A, 0x005C8D}, +{0x02F87B, 0x02F87B, 0x021DE4}, +{0x02F87C, 0x02F87C, 0x005D43}, +{0x02F87D, 0x02F87D, 0x021DE6}, +{0x02F87E, 0x02F87E, 0x005D6E}, +{0x02F87F, 0x02F87F, 0x005D6B}, +{0x02F880, 0x02F880, 0x005D7C}, +{0x02F881, 0x02F881, 0x005DE1}, +{0x02F882, 0x02F882, 0x005DE2}, +{0x02F883, 0x02F883, 0x00382F}, +{0x02F884, 0x02F884, 0x005DFD}, +{0x02F885, 0x02F885, 0x005E28}, +{0x02F886, 0x02F886, 0x005E3D}, +{0x02F887, 0x02F887, 0x005E69}, +{0x02F888, 0x02F888, 0x003862}, +{0x02F889, 0x02F889, 0x022183}, +{0x02F88A, 0x02F88A, 0x00387C}, +{0x02F88B, 0x02F88B, 0x005EB0}, +{0x02F88C, 0x02F88C, 0x005EB3}, +{0x02F88D, 0x02F88D, 0x005EB6}, +{0x02F88E, 0x02F88E, 0x005ECA}, +{0x02F88F, 0x02F88F, 0x02A392}, +{0x02F890, 0x02F890, 0x005EFE}, +{0x02F891, 0x02F892, 0x022331}, +{0x02F893, 0x02F893, 0x008201}, +{0x02F894, 0x02F895, 0x005F22}, +{0x02F896, 0x02F896, 0x0038C7}, +{0x02F897, 0x02F897, 0x0232B8}, +{0x02F898, 0x02F898, 0x0261DA}, +{0x02F899, 0x02F899, 0x005F62}, +{0x02F89A, 0x02F89A, 0x005F6B}, +{0x02F89B, 0x02F89B, 0x0038E3}, +{0x02F89C, 0x02F89C, 0x005F9A}, +{0x02F89D, 0x02F89D, 0x005FCD}, +{0x02F89E, 0x02F89E, 0x005FD7}, +{0x02F89F, 0x02F89F, 0x005FF9}, +{0x02F8A0, 0x02F8A0, 0x006081}, +{0x02F8A1, 0x02F8A1, 0x00393A}, +{0x02F8A2, 0x02F8A2, 0x00391C}, +{0x02F8A3, 0x02F8A3, 0x006094}, +{0x02F8A4, 0x02F8A4, 0x0226D4}, +{0x02F8A5, 0x02F8A5, 0x0060C7}, +{0x02F8A6, 0x02F8A6, 0x006148}, +{0x02F8A7, 0x02F8A7, 0x00614C}, +{0x02F8A8, 0x02F8A8, 0x00614E}, +{0x02F8A9, 0x02F8A9, 0x00614C}, +{0x02F8AA, 0x02F8AA, 0x00617A}, +{0x02F8AB, 0x02F8AB, 0x00618E}, +{0x02F8AC, 0x02F8AC, 0x0061B2}, +{0x02F8AD, 0x02F8AD, 0x0061A4}, +{0x02F8AE, 0x02F8AE, 0x0061AF}, +{0x02F8AF, 0x02F8AF, 0x0061DE}, +{0x02F8B0, 0x02F8B0, 0x0061F2}, +{0x02F8B1, 0x02F8B1, 0x0061F6}, +{0x02F8B2, 0x02F8B2, 0x006210}, +{0x02F8B3, 0x02F8B3, 0x00621B}, +{0x02F8B4, 0x02F8B4, 0x00625D}, +{0x02F8B5, 0x02F8B5, 0x0062B1}, +{0x02F8B6, 0x02F8B6, 0x0062D4}, +{0x02F8B7, 0x02F8B7, 0x006350}, +{0x02F8B8, 0x02F8B8, 0x022B0C}, +{0x02F8B9, 0x02F8B9, 0x00633D}, +{0x02F8BA, 0x02F8BA, 0x0062FC}, +{0x02F8BB, 0x02F8BB, 0x006368}, +{0x02F8BC, 0x02F8BC, 0x006383}, +{0x02F8BD, 0x02F8BD, 0x0063E4}, +{0x02F8BE, 0x02F8BE, 0x022BF1}, +{0x02F8BF, 0x02F8BF, 0x006422}, +{0x02F8C0, 0x02F8C0, 0x0063C5}, +{0x02F8C1, 0x02F8C1, 0x0063A9}, +{0x02F8C2, 0x02F8C2, 0x003A2E}, +{0x02F8C3, 0x02F8C3, 0x006469}, +{0x02F8C4, 0x02F8C4, 0x00647E}, +{0x02F8C5, 0x02F8C5, 0x00649D}, +{0x02F8C6, 0x02F8C6, 0x006477}, +{0x02F8C7, 0x02F8C7, 0x003A6C}, +{0x02F8C8, 0x02F8C8, 0x00654F}, +{0x02F8C9, 0x02F8C9, 0x00656C}, +{0x02F8CA, 0x02F8CA, 0x02300A}, +{0x02F8CB, 0x02F8CB, 0x0065E3}, +{0x02F8CC, 0x02F8CC, 0x0066F8}, +{0x02F8CD, 0x02F8CD, 0x006649}, +{0x02F8CE, 0x02F8CE, 0x003B19}, +{0x02F8CF, 0x02F8CF, 0x006691}, +{0x02F8D0, 0x02F8D0, 0x003B08}, +{0x02F8D1, 0x02F8D1, 0x003AE4}, +{0x02F8D2, 0x02F8D2, 0x005192}, +{0x02F8D3, 0x02F8D3, 0x005195}, +{0x02F8D4, 0x02F8D4, 0x006700}, +{0x02F8D5, 0x02F8D5, 0x00669C}, +{0x02F8D6, 0x02F8D6, 0x0080AD}, +{0x02F8D7, 0x02F8D7, 0x0043D9}, +{0x02F8D8, 0x02F8D8, 0x006717}, +{0x02F8D9, 0x02F8D9, 0x00671B}, +{0x02F8DA, 0x02F8DA, 0x006721}, +{0x02F8DB, 0x02F8DB, 0x00675E}, +{0x02F8DC, 0x02F8DC, 0x006753}, +{0x02F8DD, 0x02F8DD, 0x0233C3}, +{0x02F8DE, 0x02F8DE, 0x003B49}, +{0x02F8DF, 0x02F8DF, 0x0067FA}, +{0x02F8E0, 0x02F8E0, 0x006785}, +{0x02F8E1, 0x02F8E1, 0x006852}, +{0x02F8E2, 0x02F8E2, 0x006885}, +{0x02F8E3, 0x02F8E3, 0x02346D}, +{0x02F8E4, 0x02F8E4, 0x00688E}, +{0x02F8E5, 0x02F8E5, 0x00681F}, +{0x02F8E6, 0x02F8E6, 0x006914}, +{0x02F8E7, 0x02F8E7, 0x003B9D}, +{0x02F8E8, 0x02F8E8, 0x006942}, +{0x02F8E9, 0x02F8E9, 0x0069A3}, +{0x02F8EA, 0x02F8EA, 0x0069EA}, +{0x02F8EB, 0x02F8EB, 0x006AA8}, +{0x02F8EC, 0x02F8EC, 0x0236A3}, +{0x02F8ED, 0x02F8ED, 0x006ADB}, +{0x02F8EE, 0x02F8EE, 0x003C18}, +{0x02F8EF, 0x02F8EF, 0x006B21}, +{0x02F8F0, 0x02F8F0, 0x0238A7}, +{0x02F8F1, 0x02F8F1, 0x006B54}, +{0x02F8F2, 0x02F8F2, 0x003C4E}, +{0x02F8F3, 0x02F8F3, 0x006B72}, +{0x02F8F4, 0x02F8F4, 0x006B9F}, +{0x02F8F5, 0x02F8F5, 0x006BBA}, +{0x02F8F6, 0x02F8F6, 0x006BBB}, +{0x02F8F7, 0x02F8F7, 0x023A8D}, +{0x02F8F8, 0x02F8F8, 0x021D0B}, +{0x02F8F9, 0x02F8F9, 0x023AFA}, +{0x02F8FA, 0x02F8FA, 0x006C4E}, +{0x02F8FB, 0x02F8FB, 0x023CBC}, +{0x02F8FC, 0x02F8FC, 0x006CBF}, +{0x02F8FD, 0x02F8FD, 0x006CCD}, +{0x02F8FE, 0x02F8FE, 0x006C67}, +{0x02F8FF, 0x02F8FF, 0x006D16}, +{0x02F900, 0x02F900, 0x006D3E}, +{0x02F901, 0x02F901, 0x006D77}, +{0x02F902, 0x02F902, 0x006D41}, +{0x02F903, 0x02F903, 0x006D69}, +{0x02F904, 0x02F904, 0x006D78}, +{0x02F905, 0x02F905, 0x006D85}, +{0x02F906, 0x02F906, 0x023D1E}, +{0x02F907, 0x02F907, 0x006D34}, +{0x02F908, 0x02F908, 0x006E2F}, +{0x02F909, 0x02F909, 0x006E6E}, +{0x02F90A, 0x02F90A, 0x003D33}, +{0x02F90B, 0x02F90B, 0x006ECB}, +{0x02F90C, 0x02F90C, 0x006EC7}, +{0x02F90D, 0x02F90D, 0x023ED1}, +{0x02F90E, 0x02F90E, 0x006DF9}, +{0x02F90F, 0x02F90F, 0x006F6E}, +{0x02F910, 0x02F910, 0x023F5E}, +{0x02F911, 0x02F911, 0x023F8E}, +{0x02F912, 0x02F912, 0x006FC6}, +{0x02F913, 0x02F913, 0x007039}, +{0x02F914, 0x02F914, 0x00701E}, +{0x02F915, 0x02F915, 0x00701B}, +{0x02F916, 0x02F916, 0x003D96}, +{0x02F917, 0x02F917, 0x00704A}, +{0x02F918, 0x02F918, 0x00707D}, +{0x02F919, 0x02F919, 0x007077}, +{0x02F91A, 0x02F91A, 0x0070AD}, +{0x02F91B, 0x02F91B, 0x020525}, +{0x02F91C, 0x02F91C, 0x007145}, +{0x02F91D, 0x02F91D, 0x024263}, +{0x02F91E, 0x02F91E, 0x00719C}, +{0x02F91F, 0x02F91F, 0x0243AB}, +{0x02F920, 0x02F920, 0x007228}, +{0x02F921, 0x02F921, 0x007235}, +{0x02F922, 0x02F922, 0x007250}, +{0x02F923, 0x02F923, 0x024608}, +{0x02F924, 0x02F924, 0x007280}, +{0x02F925, 0x02F925, 0x007295}, +{0x02F926, 0x02F926, 0x024735}, +{0x02F927, 0x02F927, 0x024814}, +{0x02F928, 0x02F928, 0x00737A}, +{0x02F929, 0x02F929, 0x00738B}, +{0x02F92A, 0x02F92A, 0x003EAC}, +{0x02F92B, 0x02F92B, 0x0073A5}, +{0x02F92C, 0x02F92D, 0x003EB8}, +{0x02F92E, 0x02F92E, 0x007447}, +{0x02F92F, 0x02F92F, 0x00745C}, +{0x02F930, 0x02F930, 0x007471}, +{0x02F931, 0x02F931, 0x007485}, +{0x02F932, 0x02F932, 0x0074CA}, +{0x02F933, 0x02F933, 0x003F1B}, +{0x02F934, 0x02F934, 0x007524}, +{0x02F935, 0x02F935, 0x024C36}, +{0x02F936, 0x02F936, 0x00753E}, +{0x02F937, 0x02F937, 0x024C92}, +{0x02F938, 0x02F938, 0x007570}, +{0x02F939, 0x02F939, 0x02219F}, +{0x02F93A, 0x02F93A, 0x007610}, +{0x02F93B, 0x02F93B, 0x024FA1}, +{0x02F93C, 0x02F93C, 0x024FB8}, +{0x02F93D, 0x02F93D, 0x025044}, +{0x02F93E, 0x02F93E, 0x003FFC}, +{0x02F93F, 0x02F93F, 0x004008}, +{0x02F940, 0x02F940, 0x0076F4}, +{0x02F941, 0x02F941, 0x0250F3}, +{0x02F942, 0x02F942, 0x0250F2}, +{0x02F943, 0x02F943, 0x025119}, +{0x02F944, 0x02F944, 0x025133}, +{0x02F945, 0x02F945, 0x00771E}, +{0x02F946, 0x02F947, 0x00771F}, +{0x02F948, 0x02F948, 0x00774A}, +{0x02F949, 0x02F949, 0x004039}, +{0x02F94A, 0x02F94A, 0x00778B}, +{0x02F94B, 0x02F94B, 0x004046}, +{0x02F94C, 0x02F94C, 0x004096}, +{0x02F94D, 0x02F94D, 0x02541D}, +{0x02F94E, 0x02F94E, 0x00784E}, +{0x02F94F, 0x02F94F, 0x00788C}, +{0x02F950, 0x02F950, 0x0078CC}, +{0x02F951, 0x02F951, 0x0040E3}, +{0x02F952, 0x02F952, 0x025626}, +{0x02F953, 0x02F953, 0x007956}, +{0x02F954, 0x02F954, 0x02569A}, +{0x02F955, 0x02F955, 0x0256C5}, +{0x02F956, 0x02F956, 0x00798F}, +{0x02F957, 0x02F957, 0x0079EB}, +{0x02F958, 0x02F958, 0x00412F}, +{0x02F959, 0x02F959, 0x007A40}, +{0x02F95A, 0x02F95A, 0x007A4A}, +{0x02F95B, 0x02F95B, 0x007A4F}, +{0x02F95C, 0x02F95C, 0x02597C}, +{0x02F95D, 0x02F95E, 0x025AA7}, +{0x02F95F, 0x02F95F, 0x007AEE}, +{0x02F960, 0x02F960, 0x004202}, +{0x02F961, 0x02F961, 0x025BAB}, +{0x02F962, 0x02F962, 0x007BC6}, +{0x02F963, 0x02F963, 0x007BC9}, +{0x02F964, 0x02F964, 0x004227}, +{0x02F965, 0x02F965, 0x025C80}, +{0x02F966, 0x02F966, 0x007CD2}, +{0x02F967, 0x02F967, 0x0042A0}, +{0x02F968, 0x02F968, 0x007CE8}, +{0x02F969, 0x02F969, 0x007CE3}, +{0x02F96A, 0x02F96A, 0x007D00}, +{0x02F96B, 0x02F96B, 0x025F86}, +{0x02F96C, 0x02F96C, 0x007D63}, +{0x02F96D, 0x02F96D, 0x004301}, +{0x02F96E, 0x02F96E, 0x007DC7}, +{0x02F96F, 0x02F96F, 0x007E02}, +{0x02F970, 0x02F970, 0x007E45}, +{0x02F971, 0x02F971, 0x004334}, +{0x02F972, 0x02F972, 0x026228}, +{0x02F973, 0x02F973, 0x026247}, +{0x02F974, 0x02F974, 0x004359}, +{0x02F975, 0x02F975, 0x0262D9}, +{0x02F976, 0x02F976, 0x007F7A}, +{0x02F977, 0x02F977, 0x02633E}, +{0x02F978, 0x02F978, 0x007F95}, +{0x02F979, 0x02F979, 0x007FFA}, +{0x02F97A, 0x02F97A, 0x008005}, +{0x02F97B, 0x02F97B, 0x0264DA}, +{0x02F97C, 0x02F97C, 0x026523}, +{0x02F97D, 0x02F97D, 0x008060}, +{0x02F97E, 0x02F97E, 0x0265A8}, +{0x02F97F, 0x02F97F, 0x008070}, +{0x02F980, 0x02F980, 0x02335F}, +{0x02F981, 0x02F981, 0x0043D5}, +{0x02F982, 0x02F982, 0x0080B2}, +{0x02F983, 0x02F983, 0x008103}, +{0x02F984, 0x02F984, 0x00440B}, +{0x02F985, 0x02F985, 0x00813E}, +{0x02F986, 0x02F986, 0x005AB5}, +{0x02F987, 0x02F987, 0x0267A7}, +{0x02F988, 0x02F988, 0x0267B5}, +{0x02F989, 0x02F989, 0x023393}, +{0x02F98A, 0x02F98A, 0x02339C}, +{0x02F98B, 0x02F98B, 0x008201}, +{0x02F98C, 0x02F98C, 0x008204}, +{0x02F98D, 0x02F98D, 0x008F9E}, +{0x02F98E, 0x02F98E, 0x00446B}, +{0x02F98F, 0x02F98F, 0x008291}, +{0x02F990, 0x02F990, 0x00828B}, +{0x02F991, 0x02F991, 0x00829D}, +{0x02F992, 0x02F992, 0x0052B3}, +{0x02F993, 0x02F993, 0x0082B1}, +{0x02F994, 0x02F994, 0x0082B3}, +{0x02F995, 0x02F995, 0x0082BD}, +{0x02F996, 0x02F996, 0x0082E6}, +{0x02F997, 0x02F997, 0x026B3C}, +{0x02F998, 0x02F998, 0x0082E5}, +{0x02F999, 0x02F999, 0x00831D}, +{0x02F99A, 0x02F99A, 0x008363}, +{0x02F99B, 0x02F99B, 0x0083AD}, +{0x02F99C, 0x02F99C, 0x008323}, +{0x02F99D, 0x02F99D, 0x0083BD}, +{0x02F99E, 0x02F99E, 0x0083E7}, +{0x02F99F, 0x02F99F, 0x008457}, +{0x02F9A0, 0x02F9A0, 0x008353}, +{0x02F9A1, 0x02F9A1, 0x0083CA}, +{0x02F9A2, 0x02F9A2, 0x0083CC}, +{0x02F9A3, 0x02F9A3, 0x0083DC}, +{0x02F9A4, 0x02F9A4, 0x026C36}, +{0x02F9A5, 0x02F9A5, 0x026D6B}, +{0x02F9A6, 0x02F9A6, 0x026CD5}, +{0x02F9A7, 0x02F9A7, 0x00452B}, +{0x02F9A8, 0x02F9A8, 0x0084F1}, +{0x02F9A9, 0x02F9A9, 0x0084F3}, +{0x02F9AA, 0x02F9AA, 0x008516}, +{0x02F9AB, 0x02F9AB, 0x0273CA}, +{0x02F9AC, 0x02F9AC, 0x008564}, +{0x02F9AD, 0x02F9AD, 0x026F2C}, +{0x02F9AE, 0x02F9AE, 0x00455D}, +{0x02F9AF, 0x02F9AF, 0x004561}, +{0x02F9B0, 0x02F9B0, 0x026FB1}, +{0x02F9B1, 0x02F9B1, 0x0270D2}, +{0x02F9B2, 0x02F9B2, 0x00456B}, +{0x02F9B3, 0x02F9B3, 0x008650}, +{0x02F9B4, 0x02F9B4, 0x00865C}, +{0x02F9B5, 0x02F9B5, 0x008667}, +{0x02F9B6, 0x02F9B6, 0x008669}, +{0x02F9B7, 0x02F9B7, 0x0086A9}, +{0x02F9B8, 0x02F9B8, 0x008688}, +{0x02F9B9, 0x02F9B9, 0x00870E}, +{0x02F9BA, 0x02F9BA, 0x0086E2}, +{0x02F9BB, 0x02F9BB, 0x008779}, +{0x02F9BC, 0x02F9BC, 0x008728}, +{0x02F9BD, 0x02F9BD, 0x00876B}, +{0x02F9BE, 0x02F9BE, 0x008786}, +{0x02F9BF, 0x02F9BF, 0x0045D7}, +{0x02F9C0, 0x02F9C0, 0x0087E1}, +{0x02F9C1, 0x02F9C1, 0x008801}, +{0x02F9C2, 0x02F9C2, 0x0045F9}, +{0x02F9C3, 0x02F9C3, 0x008860}, +{0x02F9C4, 0x02F9C4, 0x008863}, +{0x02F9C5, 0x02F9C5, 0x027667}, +{0x02F9C6, 0x02F9C6, 0x0088D7}, +{0x02F9C7, 0x02F9C7, 0x0088DE}, +{0x02F9C8, 0x02F9C8, 0x004635}, +{0x02F9C9, 0x02F9C9, 0x0088FA}, +{0x02F9CA, 0x02F9CA, 0x0034BB}, +{0x02F9CB, 0x02F9CB, 0x0278AE}, +{0x02F9CC, 0x02F9CC, 0x027966}, +{0x02F9CD, 0x02F9CD, 0x0046BE}, +{0x02F9CE, 0x02F9CE, 0x0046C7}, +{0x02F9CF, 0x02F9CF, 0x008AA0}, +{0x02F9D0, 0x02F9D0, 0x008AED}, +{0x02F9D1, 0x02F9D1, 0x008B8A}, +{0x02F9D2, 0x02F9D2, 0x008C55}, +{0x02F9D3, 0x02F9D3, 0x027CA8}, +{0x02F9D4, 0x02F9D4, 0x008CAB}, +{0x02F9D5, 0x02F9D5, 0x008CC1}, +{0x02F9D6, 0x02F9D6, 0x008D1B}, +{0x02F9D7, 0x02F9D7, 0x008D77}, +{0x02F9D8, 0x02F9D8, 0x027F2F}, +{0x02F9D9, 0x02F9D9, 0x020804}, +{0x02F9DA, 0x02F9DA, 0x008DCB}, +{0x02F9DB, 0x02F9DB, 0x008DBC}, +{0x02F9DC, 0x02F9DC, 0x008DF0}, +{0x02F9DD, 0x02F9DD, 0x0208DE}, +{0x02F9DE, 0x02F9DE, 0x008ED4}, +{0x02F9DF, 0x02F9DF, 0x008F38}, +{0x02F9E0, 0x02F9E0, 0x0285D2}, +{0x02F9E1, 0x02F9E1, 0x0285ED}, +{0x02F9E2, 0x02F9E2, 0x009094}, +{0x02F9E3, 0x02F9E3, 0x0090F1}, +{0x02F9E4, 0x02F9E4, 0x009111}, +{0x02F9E5, 0x02F9E5, 0x02872E}, +{0x02F9E6, 0x02F9E6, 0x00911B}, +{0x02F9E7, 0x02F9E7, 0x009238}, +{0x02F9E8, 0x02F9E8, 0x0092D7}, +{0x02F9E9, 0x02F9E9, 0x0092D8}, +{0x02F9EA, 0x02F9EA, 0x00927C}, +{0x02F9EB, 0x02F9EB, 0x0093F9}, +{0x02F9EC, 0x02F9EC, 0x009415}, +{0x02F9ED, 0x02F9ED, 0x028BFA}, +{0x02F9EE, 0x02F9EE, 0x00958B}, +{0x02F9EF, 0x02F9EF, 0x004995}, +{0x02F9F0, 0x02F9F0, 0x0095B7}, +{0x02F9F1, 0x02F9F1, 0x028D77}, +{0x02F9F2, 0x02F9F2, 0x0049E6}, +{0x02F9F3, 0x02F9F3, 0x0096C3}, +{0x02F9F4, 0x02F9F4, 0x005DB2}, +{0x02F9F5, 0x02F9F5, 0x009723}, +{0x02F9F6, 0x02F9F6, 0x029145}, +{0x02F9F7, 0x02F9F7, 0x02921A}, +{0x02F9F8, 0x02F9F8, 0x004A6E}, +{0x02F9F9, 0x02F9F9, 0x004A76}, +{0x02F9FA, 0x02F9FA, 0x0097E0}, +{0x02F9FB, 0x02F9FB, 0x02940A}, +{0x02F9FC, 0x02F9FC, 0x004AB2}, +{0x02F9FD, 0x02F9FD, 0x029496}, +{0x02F9FE, 0x02F9FF, 0x00980B}, +{0x02FA00, 0x02FA00, 0x009829}, +{0x02FA01, 0x02FA01, 0x0295B6}, +{0x02FA02, 0x02FA02, 0x0098E2}, +{0x02FA03, 0x02FA03, 0x004B33}, +{0x02FA04, 0x02FA04, 0x009929}, +{0x02FA05, 0x02FA05, 0x0099A7}, +{0x02FA06, 0x02FA06, 0x0099C2}, +{0x02FA07, 0x02FA07, 0x0099FE}, +{0x02FA08, 0x02FA08, 0x004BCE}, +{0x02FA09, 0x02FA09, 0x029B30}, +{0x02FA0A, 0x02FA0A, 0x009B12}, +{0x02FA0B, 0x02FA0B, 0x009C40}, +{0x02FA0C, 0x02FA0C, 0x009CFD}, +{0x02FA0D, 0x02FA0D, 0x004CCE}, +{0x02FA0E, 0x02FA0E, 0x004CED}, +{0x02FA0F, 0x02FA0F, 0x009D67}, +{0x02FA10, 0x02FA10, 0x02A0CE}, +{0x02FA11, 0x02FA11, 0x004CF8}, +{0x02FA12, 0x02FA12, 0x02A105}, +{0x02FA13, 0x02FA13, 0x02A20E}, +{0x02FA14, 0x02FA14, 0x02A291}, +{0x02FA15, 0x02FA15, 0x009EBB}, +{0x02FA16, 0x02FA16, 0x004D56}, +{0x02FA17, 0x02FA17, 0x009EF9}, +{0x02FA18, 0x02FA18, 0x009EFE}, +{0x02FA19, 0x02FA19, 0x009F05}, +{0x02FA1A, 0x02FA1A, 0x009F0F}, +{0x02FA1B, 0x02FA1B, 0x009F16}, +{0x02FA1C, 0x02FA1C, 0x009F3B}, +{0x02FA1D, 0x02FA1D, 0x02A600}, }; -const std::vector> unicode_ranges_punctuation = { -{0x00000021, 0x00000023}, {0x00000025, 0x0000002A}, {0x0000002C, 0x0000002F}, {0x0000003A, 0x0000003B}, -{0x0000003F, 0x00000040}, {0x0000005B, 0x0000005D}, {0x0000005F, 0x0000005F}, {0x0000007B, 0x0000007B}, -{0x0000007D, 0x0000007D}, {0x000000A1, 0x000000A1}, {0x000000A7, 0x000000A7}, {0x000000AB, 0x000000AB}, -{0x000000B6, 0x000000B7}, {0x000000BB, 0x000000BB}, {0x000000BF, 0x000000BF}, {0x0000037E, 0x0000037E}, -{0x00000387, 0x00000387}, {0x0000055A, 0x0000055F}, {0x00000589, 0x0000058A}, {0x000005BE, 0x000005BE}, -{0x000005C0, 0x000005C0}, {0x000005C3, 0x000005C3}, {0x000005C6, 0x000005C6}, {0x000005F3, 0x000005F4}, -{0x00000609, 0x0000060A}, {0x0000060C, 0x0000060D}, {0x0000061B, 0x0000061B}, {0x0000061D, 0x0000061F}, -{0x0000066A, 0x0000066D}, {0x000006D4, 0x000006D4}, {0x00000700, 0x0000070D}, {0x000007F7, 0x000007F9}, -{0x00000830, 0x0000083E}, {0x0000085E, 0x0000085E}, {0x00000964, 0x00000965}, {0x00000970, 0x00000970}, -{0x000009FD, 0x000009FD}, {0x00000A76, 0x00000A76}, {0x00000AF0, 0x00000AF0}, {0x00000C77, 0x00000C77}, -{0x00000C84, 0x00000C84}, {0x00000DF4, 0x00000DF4}, {0x00000E4F, 0x00000E4F}, {0x00000E5A, 0x00000E5B}, -{0x00000F04, 0x00000F12}, {0x00000F14, 0x00000F14}, {0x00000F3A, 0x00000F3D}, {0x00000F85, 0x00000F85}, -{0x00000FD0, 0x00000FD4}, {0x00000FD9, 0x00000FDA}, {0x0000104A, 0x0000104F}, {0x000010FB, 0x000010FB}, -{0x00001360, 0x00001368}, {0x00001400, 0x00001400}, {0x0000166E, 0x0000166E}, {0x0000169B, 0x0000169C}, -{0x000016EB, 0x000016ED}, {0x00001735, 0x00001736}, {0x000017D4, 0x000017D6}, {0x000017D8, 0x000017DA}, -{0x00001800, 0x0000180A}, {0x00001944, 0x00001945}, {0x00001A1E, 0x00001A1F}, {0x00001AA0, 0x00001AA6}, -{0x00001AA8, 0x00001AAD}, {0x00001B5A, 0x00001B60}, {0x00001B7D, 0x00001B7E}, {0x00001BFC, 0x00001BFF}, -{0x00001C3B, 0x00001C3F}, {0x00001C7E, 0x00001C7F}, {0x00001CC0, 0x00001CC7}, {0x00001CD3, 0x00001CD3}, -{0x00002010, 0x00002027}, {0x00002030, 0x00002043}, {0x00002045, 0x00002051}, {0x00002053, 0x0000205E}, -{0x0000207D, 0x0000207E}, {0x0000208D, 0x0000208E}, {0x00002308, 0x0000230B}, {0x00002329, 0x0000232A}, -{0x00002768, 0x00002775}, {0x000027C5, 0x000027C6}, {0x000027E6, 0x000027EF}, {0x00002983, 0x00002998}, -{0x000029D8, 0x000029DB}, {0x000029FC, 0x000029FD}, {0x00002CF9, 0x00002CFC}, {0x00002CFE, 0x00002CFF}, -{0x00002D70, 0x00002D70}, {0x00002E00, 0x00002E2E}, {0x00002E30, 0x00002E4F}, {0x00002E52, 0x00002E5D}, -{0x00003001, 0x00003003}, {0x00003008, 0x00003011}, {0x00003014, 0x0000301F}, {0x00003030, 0x00003030}, -{0x0000303D, 0x0000303D}, {0x000030A0, 0x000030A0}, {0x000030FB, 0x000030FB}, {0x0000A4FE, 0x0000A4FF}, -{0x0000A60D, 0x0000A60F}, {0x0000A673, 0x0000A673}, {0x0000A67E, 0x0000A67E}, {0x0000A6F2, 0x0000A6F7}, -{0x0000A874, 0x0000A877}, {0x0000A8CE, 0x0000A8CF}, {0x0000A8F8, 0x0000A8FA}, {0x0000A8FC, 0x0000A8FC}, -{0x0000A92E, 0x0000A92F}, {0x0000A95F, 0x0000A95F}, {0x0000A9C1, 0x0000A9CD}, {0x0000A9DE, 0x0000A9DF}, -{0x0000AA5C, 0x0000AA5F}, {0x0000AADE, 0x0000AADF}, {0x0000AAF0, 0x0000AAF1}, {0x0000ABEB, 0x0000ABEB}, -{0x0000FD3E, 0x0000FD3F}, {0x0000FE10, 0x0000FE19}, {0x0000FE30, 0x0000FE52}, {0x0000FE54, 0x0000FE61}, -{0x0000FE63, 0x0000FE63}, {0x0000FE68, 0x0000FE68}, {0x0000FE6A, 0x0000FE6B}, {0x0000FF01, 0x0000FF03}, -{0x0000FF05, 0x0000FF0A}, {0x0000FF0C, 0x0000FF0F}, {0x0000FF1A, 0x0000FF1B}, {0x0000FF1F, 0x0000FF20}, -{0x0000FF3B, 0x0000FF3D}, {0x0000FF3F, 0x0000FF3F}, {0x0000FF5B, 0x0000FF5B}, {0x0000FF5D, 0x0000FF5D}, -{0x0000FF5F, 0x0000FF65}, {0x00010100, 0x00010102}, {0x0001039F, 0x0001039F}, {0x000103D0, 0x000103D0}, -{0x0001056F, 0x0001056F}, {0x00010857, 0x00010857}, {0x0001091F, 0x0001091F}, {0x0001093F, 0x0001093F}, -{0x00010A50, 0x00010A58}, {0x00010A7F, 0x00010A7F}, {0x00010AF0, 0x00010AF6}, {0x00010B39, 0x00010B3F}, -{0x00010B99, 0x00010B9C}, {0x00010EAD, 0x00010EAD}, {0x00010F55, 0x00010F59}, {0x00010F86, 0x00010F89}, -{0x00011047, 0x0001104D}, {0x000110BB, 0x000110BC}, {0x000110BE, 0x000110C1}, {0x00011140, 0x00011143}, -{0x00011174, 0x00011175}, {0x000111C5, 0x000111C8}, {0x000111CD, 0x000111CD}, {0x000111DB, 0x000111DB}, -{0x000111DD, 0x000111DF}, {0x00011238, 0x0001123D}, {0x000112A9, 0x000112A9}, {0x0001144B, 0x0001144F}, -{0x0001145A, 0x0001145B}, {0x0001145D, 0x0001145D}, {0x000114C6, 0x000114C6}, {0x000115C1, 0x000115D7}, -{0x00011641, 0x00011643}, {0x00011660, 0x0001166C}, {0x000116B9, 0x000116B9}, {0x0001173C, 0x0001173E}, -{0x0001183B, 0x0001183B}, {0x00011944, 0x00011946}, {0x000119E2, 0x000119E2}, {0x00011A3F, 0x00011A46}, -{0x00011A9A, 0x00011A9C}, {0x00011A9E, 0x00011AA2}, {0x00011B00, 0x00011B09}, {0x00011C41, 0x00011C45}, -{0x00011C70, 0x00011C71}, {0x00011EF7, 0x00011EF8}, {0x00011F43, 0x00011F4F}, {0x00011FFF, 0x00011FFF}, -{0x00012470, 0x00012474}, {0x00012FF1, 0x00012FF2}, {0x00016A6E, 0x00016A6F}, {0x00016AF5, 0x00016AF5}, -{0x00016B37, 0x00016B3B}, {0x00016B44, 0x00016B44}, {0x00016E97, 0x00016E9A}, {0x00016FE2, 0x00016FE2}, -{0x0001BC9F, 0x0001BC9F}, {0x0001DA87, 0x0001DA8B}, {0x0001E95E, 0x0001E95F}, -}; - -const std::vector> unicode_ranges_symbol = { -{0x00000024, 0x00000024}, {0x0000002B, 0x0000002B}, {0x0000003C, 0x0000003E}, {0x0000005E, 0x0000005E}, -{0x00000060, 0x00000060}, {0x0000007C, 0x0000007C}, {0x0000007E, 0x0000007E}, {0x000000A2, 0x000000A6}, -{0x000000A8, 0x000000A9}, {0x000000AC, 0x000000AC}, {0x000000AE, 0x000000B1}, {0x000000B4, 0x000000B4}, -{0x000000B8, 0x000000B8}, {0x000000D7, 0x000000D7}, {0x000000F7, 0x000000F7}, {0x000002C2, 0x000002C5}, -{0x000002D2, 0x000002DF}, {0x000002E5, 0x000002EB}, {0x000002ED, 0x000002ED}, {0x000002EF, 0x000002FF}, -{0x00000375, 0x00000375}, {0x00000384, 0x00000385}, {0x000003F6, 0x000003F6}, {0x00000482, 0x00000482}, -{0x0000058D, 0x0000058F}, {0x00000606, 0x00000608}, {0x0000060B, 0x0000060B}, {0x0000060E, 0x0000060F}, -{0x000006DE, 0x000006DE}, {0x000006E9, 0x000006E9}, {0x000006FD, 0x000006FE}, {0x000007F6, 0x000007F6}, -{0x000007FE, 0x000007FF}, {0x00000888, 0x00000888}, {0x000009F2, 0x000009F3}, {0x000009FA, 0x000009FB}, -{0x00000AF1, 0x00000AF1}, {0x00000B70, 0x00000B70}, {0x00000BF3, 0x00000BFA}, {0x00000C7F, 0x00000C7F}, -{0x00000D4F, 0x00000D4F}, {0x00000D79, 0x00000D79}, {0x00000E3F, 0x00000E3F}, {0x00000F01, 0x00000F03}, -{0x00000F13, 0x00000F13}, {0x00000F15, 0x00000F17}, {0x00000F1A, 0x00000F1F}, {0x00000F34, 0x00000F34}, -{0x00000F36, 0x00000F36}, {0x00000F38, 0x00000F38}, {0x00000FBE, 0x00000FC5}, {0x00000FC7, 0x00000FCC}, -{0x00000FCE, 0x00000FCF}, {0x00000FD5, 0x00000FD8}, {0x0000109E, 0x0000109F}, {0x00001390, 0x00001399}, -{0x0000166D, 0x0000166D}, {0x000017DB, 0x000017DB}, {0x00001940, 0x00001940}, {0x000019DE, 0x000019FF}, -{0x00001B61, 0x00001B6A}, {0x00001B74, 0x00001B7C}, {0x00001FBD, 0x00001FBD}, {0x00001FBF, 0x00001FC1}, -{0x00001FCD, 0x00001FCF}, {0x00001FDD, 0x00001FDF}, {0x00001FED, 0x00001FEF}, {0x00001FFD, 0x00001FFE}, -{0x00002044, 0x00002044}, {0x00002052, 0x00002052}, {0x0000207A, 0x0000207C}, {0x0000208A, 0x0000208C}, -{0x000020A0, 0x000020C0}, {0x00002100, 0x00002101}, {0x00002103, 0x00002106}, {0x00002108, 0x00002109}, -{0x00002114, 0x00002114}, {0x00002116, 0x00002118}, {0x0000211E, 0x00002123}, {0x00002125, 0x00002125}, -{0x00002127, 0x00002127}, {0x00002129, 0x00002129}, {0x0000212E, 0x0000212E}, {0x0000213A, 0x0000213B}, -{0x00002140, 0x00002144}, {0x0000214A, 0x0000214D}, {0x0000214F, 0x0000214F}, {0x0000218A, 0x0000218B}, -{0x00002190, 0x00002307}, {0x0000230C, 0x00002328}, {0x0000232B, 0x00002426}, {0x00002440, 0x0000244A}, -{0x0000249C, 0x000024E9}, {0x00002500, 0x00002767}, {0x00002794, 0x000027C4}, {0x000027C7, 0x000027E5}, -{0x000027F0, 0x00002982}, {0x00002999, 0x000029D7}, {0x000029DC, 0x000029FB}, {0x000029FE, 0x00002B73}, -{0x00002B76, 0x00002B95}, {0x00002B97, 0x00002BFF}, {0x00002CE5, 0x00002CEA}, {0x00002E50, 0x00002E51}, -{0x00002E80, 0x00002E99}, {0x00002E9B, 0x00002EF3}, {0x00002F00, 0x00002FD5}, {0x00002FF0, 0x00002FFF}, -{0x00003004, 0x00003004}, {0x00003012, 0x00003013}, {0x00003020, 0x00003020}, {0x00003036, 0x00003037}, -{0x0000303E, 0x0000303F}, {0x0000309B, 0x0000309C}, {0x00003190, 0x00003191}, {0x00003196, 0x0000319F}, -{0x000031C0, 0x000031E3}, {0x000031EF, 0x000031EF}, {0x00003200, 0x0000321E}, {0x0000322A, 0x00003247}, -{0x00003250, 0x00003250}, {0x00003260, 0x0000327F}, {0x0000328A, 0x000032B0}, {0x000032C0, 0x000033FF}, -{0x00004DC0, 0x00004DFF}, {0x0000A490, 0x0000A4C6}, {0x0000A700, 0x0000A716}, {0x0000A720, 0x0000A721}, -{0x0000A789, 0x0000A78A}, {0x0000A828, 0x0000A82B}, {0x0000A836, 0x0000A839}, {0x0000AA77, 0x0000AA79}, -{0x0000AB5B, 0x0000AB5B}, {0x0000AB6A, 0x0000AB6B}, {0x0000FB29, 0x0000FB29}, {0x0000FBB2, 0x0000FBC2}, -{0x0000FD40, 0x0000FD4F}, {0x0000FDCF, 0x0000FDCF}, {0x0000FDFC, 0x0000FDFF}, {0x0000FE62, 0x0000FE62}, -{0x0000FE64, 0x0000FE66}, {0x0000FE69, 0x0000FE69}, {0x0000FF04, 0x0000FF04}, {0x0000FF0B, 0x0000FF0B}, -{0x0000FF1C, 0x0000FF1E}, {0x0000FF3E, 0x0000FF3E}, {0x0000FF40, 0x0000FF40}, {0x0000FF5C, 0x0000FF5C}, -{0x0000FF5E, 0x0000FF5E}, {0x0000FFE0, 0x0000FFE6}, {0x0000FFE8, 0x0000FFEE}, {0x0000FFFC, 0x0000FFFD}, -{0x00010137, 0x0001013F}, {0x00010179, 0x00010189}, {0x0001018C, 0x0001018E}, {0x00010190, 0x0001019C}, -{0x000101A0, 0x000101A0}, {0x000101D0, 0x000101FC}, {0x00010877, 0x00010878}, {0x00010AC8, 0x00010AC8}, -{0x0001173F, 0x0001173F}, {0x00011FD5, 0x00011FF1}, {0x00016B3C, 0x00016B3F}, {0x00016B45, 0x00016B45}, -{0x0001BC9C, 0x0001BC9C}, {0x0001CF50, 0x0001CFC3}, {0x0001D000, 0x0001D0F5}, {0x0001D100, 0x0001D126}, -{0x0001D129, 0x0001D164}, {0x0001D16A, 0x0001D16C}, {0x0001D183, 0x0001D184}, {0x0001D18C, 0x0001D1A9}, -{0x0001D1AE, 0x0001D1EA}, {0x0001D200, 0x0001D241}, {0x0001D245, 0x0001D245}, {0x0001D300, 0x0001D356}, -{0x0001D6C1, 0x0001D6C1}, {0x0001D6DB, 0x0001D6DB}, {0x0001D6FB, 0x0001D6FB}, {0x0001D715, 0x0001D715}, -{0x0001D735, 0x0001D735}, {0x0001D74F, 0x0001D74F}, {0x0001D76F, 0x0001D76F}, {0x0001D789, 0x0001D789}, -{0x0001D7A9, 0x0001D7A9}, {0x0001D7C3, 0x0001D7C3}, {0x0001D800, 0x0001D9FF}, {0x0001DA37, 0x0001DA3A}, -{0x0001DA6D, 0x0001DA74}, {0x0001DA76, 0x0001DA83}, {0x0001DA85, 0x0001DA86}, {0x0001E14F, 0x0001E14F}, -{0x0001E2FF, 0x0001E2FF}, {0x0001ECAC, 0x0001ECAC}, {0x0001ECB0, 0x0001ECB0}, {0x0001ED2E, 0x0001ED2E}, -{0x0001EEF0, 0x0001EEF1}, {0x0001F000, 0x0001F02B}, {0x0001F030, 0x0001F093}, {0x0001F0A0, 0x0001F0AE}, -{0x0001F0B1, 0x0001F0BF}, {0x0001F0C1, 0x0001F0CF}, {0x0001F0D1, 0x0001F0F5}, {0x0001F10D, 0x0001F1AD}, -{0x0001F1E6, 0x0001F202}, {0x0001F210, 0x0001F23B}, {0x0001F240, 0x0001F248}, {0x0001F250, 0x0001F251}, -{0x0001F260, 0x0001F265}, {0x0001F300, 0x0001F6D7}, {0x0001F6DC, 0x0001F6EC}, {0x0001F6F0, 0x0001F6FC}, -{0x0001F700, 0x0001F776}, {0x0001F77B, 0x0001F7D9}, {0x0001F7E0, 0x0001F7EB}, {0x0001F7F0, 0x0001F7F0}, -{0x0001F800, 0x0001F80B}, {0x0001F810, 0x0001F847}, {0x0001F850, 0x0001F859}, {0x0001F860, 0x0001F887}, -{0x0001F890, 0x0001F8AD}, {0x0001F8B0, 0x0001F8B1}, {0x0001F900, 0x0001FA53}, {0x0001FA60, 0x0001FA6D}, -{0x0001FA70, 0x0001FA7C}, {0x0001FA80, 0x0001FA88}, {0x0001FA90, 0x0001FABD}, {0x0001FABF, 0x0001FAC5}, -{0x0001FACE, 0x0001FADB}, {0x0001FAE0, 0x0001FAE8}, {0x0001FAF0, 0x0001FAF8}, {0x0001FB00, 0x0001FB92}, -{0x0001FB94, 0x0001FBCA}, -}; - -const std::vector> unicode_ranges_control = { -{0x00000000, 0x0000001F}, {0x0000007F, 0x0000009F}, {0x000000AD, 0x000000AD}, {0x00000378, 0x00000379}, -{0x00000380, 0x00000383}, {0x0000038B, 0x0000038B}, {0x0000038D, 0x0000038D}, {0x000003A2, 0x000003A2}, -{0x00000530, 0x00000530}, {0x00000557, 0x00000558}, {0x0000058B, 0x0000058C}, {0x00000590, 0x00000590}, -{0x000005C8, 0x000005CF}, {0x000005EB, 0x000005EE}, {0x000005F5, 0x00000605}, {0x0000061C, 0x0000061C}, -{0x000006DD, 0x000006DD}, {0x0000070E, 0x0000070F}, {0x0000074B, 0x0000074C}, {0x000007B2, 0x000007BF}, -{0x000007FB, 0x000007FC}, {0x0000082E, 0x0000082F}, {0x0000083F, 0x0000083F}, {0x0000085C, 0x0000085D}, -{0x0000085F, 0x0000085F}, {0x0000086B, 0x0000086F}, {0x0000088F, 0x00000897}, {0x000008E2, 0x000008E2}, -{0x00000984, 0x00000984}, {0x0000098D, 0x0000098E}, {0x00000991, 0x00000992}, {0x000009A9, 0x000009A9}, -{0x000009B1, 0x000009B1}, {0x000009B3, 0x000009B5}, {0x000009BA, 0x000009BB}, {0x000009C5, 0x000009C6}, -{0x000009C9, 0x000009CA}, {0x000009CF, 0x000009D6}, {0x000009D8, 0x000009DB}, {0x000009DE, 0x000009DE}, -{0x000009E4, 0x000009E5}, {0x000009FF, 0x00000A00}, {0x00000A04, 0x00000A04}, {0x00000A0B, 0x00000A0E}, -{0x00000A11, 0x00000A12}, {0x00000A29, 0x00000A29}, {0x00000A31, 0x00000A31}, {0x00000A34, 0x00000A34}, -{0x00000A37, 0x00000A37}, {0x00000A3A, 0x00000A3B}, {0x00000A3D, 0x00000A3D}, {0x00000A43, 0x00000A46}, -{0x00000A49, 0x00000A4A}, {0x00000A4E, 0x00000A50}, {0x00000A52, 0x00000A58}, {0x00000A5D, 0x00000A5D}, -{0x00000A5F, 0x00000A65}, {0x00000A77, 0x00000A80}, {0x00000A84, 0x00000A84}, {0x00000A8E, 0x00000A8E}, -{0x00000A92, 0x00000A92}, {0x00000AA9, 0x00000AA9}, {0x00000AB1, 0x00000AB1}, {0x00000AB4, 0x00000AB4}, -{0x00000ABA, 0x00000ABB}, {0x00000AC6, 0x00000AC6}, {0x00000ACA, 0x00000ACA}, {0x00000ACE, 0x00000ACF}, -{0x00000AD1, 0x00000ADF}, {0x00000AE4, 0x00000AE5}, {0x00000AF2, 0x00000AF8}, {0x00000B00, 0x00000B00}, -{0x00000B04, 0x00000B04}, {0x00000B0D, 0x00000B0E}, {0x00000B11, 0x00000B12}, {0x00000B29, 0x00000B29}, -{0x00000B31, 0x00000B31}, {0x00000B34, 0x00000B34}, {0x00000B3A, 0x00000B3B}, {0x00000B45, 0x00000B46}, -{0x00000B49, 0x00000B4A}, {0x00000B4E, 0x00000B54}, {0x00000B58, 0x00000B5B}, {0x00000B5E, 0x00000B5E}, -{0x00000B64, 0x00000B65}, {0x00000B78, 0x00000B81}, {0x00000B84, 0x00000B84}, {0x00000B8B, 0x00000B8D}, -{0x00000B91, 0x00000B91}, {0x00000B96, 0x00000B98}, {0x00000B9B, 0x00000B9B}, {0x00000B9D, 0x00000B9D}, -{0x00000BA0, 0x00000BA2}, {0x00000BA5, 0x00000BA7}, {0x00000BAB, 0x00000BAD}, {0x00000BBA, 0x00000BBD}, -{0x00000BC3, 0x00000BC5}, {0x00000BC9, 0x00000BC9}, {0x00000BCE, 0x00000BCF}, {0x00000BD1, 0x00000BD6}, -{0x00000BD8, 0x00000BE5}, {0x00000BFB, 0x00000BFF}, {0x00000C0D, 0x00000C0D}, {0x00000C11, 0x00000C11}, -{0x00000C29, 0x00000C29}, {0x00000C3A, 0x00000C3B}, {0x00000C45, 0x00000C45}, {0x00000C49, 0x00000C49}, -{0x00000C4E, 0x00000C54}, {0x00000C57, 0x00000C57}, {0x00000C5B, 0x00000C5C}, {0x00000C5E, 0x00000C5F}, -{0x00000C64, 0x00000C65}, {0x00000C70, 0x00000C76}, {0x00000C8D, 0x00000C8D}, {0x00000C91, 0x00000C91}, -{0x00000CA9, 0x00000CA9}, {0x00000CB4, 0x00000CB4}, {0x00000CBA, 0x00000CBB}, {0x00000CC5, 0x00000CC5}, -{0x00000CC9, 0x00000CC9}, {0x00000CCE, 0x00000CD4}, {0x00000CD7, 0x00000CDC}, {0x00000CDF, 0x00000CDF}, -{0x00000CE4, 0x00000CE5}, {0x00000CF0, 0x00000CF0}, {0x00000CF4, 0x00000CFF}, {0x00000D0D, 0x00000D0D}, -{0x00000D11, 0x00000D11}, {0x00000D45, 0x00000D45}, {0x00000D49, 0x00000D49}, {0x00000D50, 0x00000D53}, -{0x00000D64, 0x00000D65}, {0x00000D80, 0x00000D80}, {0x00000D84, 0x00000D84}, {0x00000D97, 0x00000D99}, -{0x00000DB2, 0x00000DB2}, {0x00000DBC, 0x00000DBC}, {0x00000DBE, 0x00000DBF}, {0x00000DC7, 0x00000DC9}, -{0x00000DCB, 0x00000DCE}, {0x00000DD5, 0x00000DD5}, {0x00000DD7, 0x00000DD7}, {0x00000DE0, 0x00000DE5}, -{0x00000DF0, 0x00000DF1}, {0x00000DF5, 0x00000E00}, {0x00000E3B, 0x00000E3E}, {0x00000E5C, 0x00000E80}, -{0x00000E83, 0x00000E83}, {0x00000E85, 0x00000E85}, {0x00000E8B, 0x00000E8B}, {0x00000EA4, 0x00000EA4}, -{0x00000EA6, 0x00000EA6}, {0x00000EBE, 0x00000EBF}, {0x00000EC5, 0x00000EC5}, {0x00000EC7, 0x00000EC7}, -{0x00000ECF, 0x00000ECF}, {0x00000EDA, 0x00000EDB}, {0x00000EE0, 0x00000EFF}, {0x00000F48, 0x00000F48}, -{0x00000F6D, 0x00000F70}, {0x00000F98, 0x00000F98}, {0x00000FBD, 0x00000FBD}, {0x00000FCD, 0x00000FCD}, -{0x00000FDB, 0x00000FFF}, {0x000010C6, 0x000010C6}, {0x000010C8, 0x000010CC}, {0x000010CE, 0x000010CF}, -{0x00001249, 0x00001249}, {0x0000124E, 0x0000124F}, {0x00001257, 0x00001257}, {0x00001259, 0x00001259}, -{0x0000125E, 0x0000125F}, {0x00001289, 0x00001289}, {0x0000128E, 0x0000128F}, {0x000012B1, 0x000012B1}, -{0x000012B6, 0x000012B7}, {0x000012BF, 0x000012BF}, {0x000012C1, 0x000012C1}, {0x000012C6, 0x000012C7}, -{0x000012D7, 0x000012D7}, {0x00001311, 0x00001311}, {0x00001316, 0x00001317}, {0x0000135B, 0x0000135C}, -{0x0000137D, 0x0000137F}, {0x0000139A, 0x0000139F}, {0x000013F6, 0x000013F7}, {0x000013FE, 0x000013FF}, -{0x0000169D, 0x0000169F}, {0x000016F9, 0x000016FF}, {0x00001716, 0x0000171E}, {0x00001737, 0x0000173F}, -{0x00001754, 0x0000175F}, {0x0000176D, 0x0000176D}, {0x00001771, 0x00001771}, {0x00001774, 0x0000177F}, -{0x000017DE, 0x000017DF}, {0x000017EA, 0x000017EF}, {0x000017FA, 0x000017FF}, {0x0000180E, 0x0000180E}, -{0x0000181A, 0x0000181F}, {0x00001879, 0x0000187F}, {0x000018AB, 0x000018AF}, {0x000018F6, 0x000018FF}, -{0x0000191F, 0x0000191F}, {0x0000192C, 0x0000192F}, {0x0000193C, 0x0000193F}, {0x00001941, 0x00001943}, -{0x0000196E, 0x0000196F}, {0x00001975, 0x0000197F}, {0x000019AC, 0x000019AF}, {0x000019CA, 0x000019CF}, -{0x000019DB, 0x000019DD}, {0x00001A1C, 0x00001A1D}, {0x00001A5F, 0x00001A5F}, {0x00001A7D, 0x00001A7E}, -{0x00001A8A, 0x00001A8F}, {0x00001A9A, 0x00001A9F}, {0x00001AAE, 0x00001AAF}, {0x00001ACF, 0x00001AFF}, -{0x00001B4D, 0x00001B4F}, {0x00001B7F, 0x00001B7F}, {0x00001BF4, 0x00001BFB}, {0x00001C38, 0x00001C3A}, -{0x00001C4A, 0x00001C4C}, {0x00001C89, 0x00001C8F}, {0x00001CBB, 0x00001CBC}, {0x00001CC8, 0x00001CCF}, -{0x00001CFB, 0x00001CFF}, {0x00001F16, 0x00001F17}, {0x00001F1E, 0x00001F1F}, {0x00001F46, 0x00001F47}, -{0x00001F4E, 0x00001F4F}, {0x00001F58, 0x00001F58}, {0x00001F5A, 0x00001F5A}, {0x00001F5C, 0x00001F5C}, -{0x00001F5E, 0x00001F5E}, {0x00001F7E, 0x00001F7F}, {0x00001FB5, 0x00001FB5}, {0x00001FC5, 0x00001FC5}, -{0x00001FD4, 0x00001FD5}, {0x00001FDC, 0x00001FDC}, {0x00001FF0, 0x00001FF1}, {0x00001FF5, 0x00001FF5}, -{0x00001FFF, 0x00001FFF}, {0x0000200B, 0x0000200F}, {0x0000202A, 0x0000202E}, {0x00002060, 0x0000206F}, -{0x00002072, 0x00002073}, {0x0000208F, 0x0000208F}, {0x0000209D, 0x0000209F}, {0x000020C1, 0x000020CF}, -{0x000020F1, 0x000020FF}, {0x0000218C, 0x0000218F}, {0x00002427, 0x0000243F}, {0x0000244B, 0x0000245F}, -{0x00002B74, 0x00002B75}, {0x00002B96, 0x00002B96}, {0x00002CF4, 0x00002CF8}, {0x00002D26, 0x00002D26}, -{0x00002D28, 0x00002D2C}, {0x00002D2E, 0x00002D2F}, {0x00002D68, 0x00002D6E}, {0x00002D71, 0x00002D7E}, -{0x00002D97, 0x00002D9F}, {0x00002DA7, 0x00002DA7}, {0x00002DAF, 0x00002DAF}, {0x00002DB7, 0x00002DB7}, -{0x00002DBF, 0x00002DBF}, {0x00002DC7, 0x00002DC7}, {0x00002DCF, 0x00002DCF}, {0x00002DD7, 0x00002DD7}, -{0x00002DDF, 0x00002DDF}, {0x00002E5E, 0x00002E7F}, {0x00002E9A, 0x00002E9A}, {0x00002EF4, 0x00002EFF}, -{0x00002FD6, 0x00002FEF}, {0x00003040, 0x00003040}, {0x00003097, 0x00003098}, {0x00003100, 0x00003104}, -{0x00003130, 0x00003130}, {0x0000318F, 0x0000318F}, {0x000031E4, 0x000031EE}, {0x0000321F, 0x0000321F}, -{0x0000A48D, 0x0000A48F}, {0x0000A4C7, 0x0000A4CF}, {0x0000A62C, 0x0000A63F}, {0x0000A6F8, 0x0000A6FF}, -{0x0000A7CB, 0x0000A7CF}, {0x0000A7D2, 0x0000A7D2}, {0x0000A7D4, 0x0000A7D4}, {0x0000A7DA, 0x0000A7F1}, -{0x0000A82D, 0x0000A82F}, {0x0000A83A, 0x0000A83F}, {0x0000A878, 0x0000A87F}, {0x0000A8C6, 0x0000A8CD}, -{0x0000A8DA, 0x0000A8DF}, {0x0000A954, 0x0000A95E}, {0x0000A97D, 0x0000A97F}, {0x0000A9CE, 0x0000A9CE}, -{0x0000A9DA, 0x0000A9DD}, {0x0000A9FF, 0x0000A9FF}, {0x0000AA37, 0x0000AA3F}, {0x0000AA4E, 0x0000AA4F}, -{0x0000AA5A, 0x0000AA5B}, {0x0000AAC3, 0x0000AADA}, {0x0000AAF7, 0x0000AB00}, {0x0000AB07, 0x0000AB08}, -{0x0000AB0F, 0x0000AB10}, {0x0000AB17, 0x0000AB1F}, {0x0000AB27, 0x0000AB27}, {0x0000AB2F, 0x0000AB2F}, -{0x0000AB6C, 0x0000AB6F}, {0x0000ABEE, 0x0000ABEF}, {0x0000ABFA, 0x0000ABFF}, {0x0000D7A4, 0x0000D7AF}, -{0x0000D7C7, 0x0000D7CA}, {0x0000D7FC, 0x0000F8FF}, {0x0000FA6E, 0x0000FA6F}, {0x0000FADA, 0x0000FAFF}, -{0x0000FB07, 0x0000FB12}, {0x0000FB18, 0x0000FB1C}, {0x0000FB37, 0x0000FB37}, {0x0000FB3D, 0x0000FB3D}, -{0x0000FB3F, 0x0000FB3F}, {0x0000FB42, 0x0000FB42}, {0x0000FB45, 0x0000FB45}, {0x0000FBC3, 0x0000FBD2}, -{0x0000FD90, 0x0000FD91}, {0x0000FDC8, 0x0000FDCE}, {0x0000FDD0, 0x0000FDEF}, {0x0000FE1A, 0x0000FE1F}, -{0x0000FE53, 0x0000FE53}, {0x0000FE67, 0x0000FE67}, {0x0000FE6C, 0x0000FE6F}, {0x0000FE75, 0x0000FE75}, -{0x0000FEFD, 0x0000FF00}, {0x0000FFBF, 0x0000FFC1}, {0x0000FFC8, 0x0000FFC9}, {0x0000FFD0, 0x0000FFD1}, -{0x0000FFD8, 0x0000FFD9}, {0x0000FFDD, 0x0000FFDF}, {0x0000FFE7, 0x0000FFE7}, {0x0000FFEF, 0x0000FFFB}, -{0x0000FFFE, 0x0000FFFF}, {0x0001000C, 0x0001000C}, {0x00010027, 0x00010027}, {0x0001003B, 0x0001003B}, -{0x0001003E, 0x0001003E}, {0x0001004E, 0x0001004F}, {0x0001005E, 0x0001007F}, {0x000100FB, 0x000100FF}, -{0x00010103, 0x00010106}, {0x00010134, 0x00010136}, {0x0001018F, 0x0001018F}, {0x0001019D, 0x0001019F}, -{0x000101A1, 0x000101CF}, {0x000101FE, 0x0001027F}, {0x0001029D, 0x0001029F}, {0x000102D1, 0x000102DF}, -{0x000102FC, 0x000102FF}, {0x00010324, 0x0001032C}, {0x0001034B, 0x0001034F}, {0x0001037B, 0x0001037F}, -{0x0001039E, 0x0001039E}, {0x000103C4, 0x000103C7}, {0x000103D6, 0x000103FF}, {0x0001049E, 0x0001049F}, -{0x000104AA, 0x000104AF}, {0x000104D4, 0x000104D7}, {0x000104FC, 0x000104FF}, {0x00010528, 0x0001052F}, -{0x00010564, 0x0001056E}, {0x0001057B, 0x0001057B}, {0x0001058B, 0x0001058B}, {0x00010593, 0x00010593}, -{0x00010596, 0x00010596}, {0x000105A2, 0x000105A2}, {0x000105B2, 0x000105B2}, {0x000105BA, 0x000105BA}, -{0x000105BD, 0x000105FF}, {0x00010737, 0x0001073F}, {0x00010756, 0x0001075F}, {0x00010768, 0x0001077F}, -{0x00010786, 0x00010786}, {0x000107B1, 0x000107B1}, {0x000107BB, 0x000107FF}, {0x00010806, 0x00010807}, -{0x00010809, 0x00010809}, {0x00010836, 0x00010836}, {0x00010839, 0x0001083B}, {0x0001083D, 0x0001083E}, -{0x00010856, 0x00010856}, {0x0001089F, 0x000108A6}, {0x000108B0, 0x000108DF}, {0x000108F3, 0x000108F3}, -{0x000108F6, 0x000108FA}, {0x0001091C, 0x0001091E}, {0x0001093A, 0x0001093E}, {0x00010940, 0x0001097F}, -{0x000109B8, 0x000109BB}, {0x000109D0, 0x000109D1}, {0x00010A04, 0x00010A04}, {0x00010A07, 0x00010A0B}, -{0x00010A14, 0x00010A14}, {0x00010A18, 0x00010A18}, {0x00010A36, 0x00010A37}, {0x00010A3B, 0x00010A3E}, -{0x00010A49, 0x00010A4F}, {0x00010A59, 0x00010A5F}, {0x00010AA0, 0x00010ABF}, {0x00010AE7, 0x00010AEA}, -{0x00010AF7, 0x00010AFF}, {0x00010B36, 0x00010B38}, {0x00010B56, 0x00010B57}, {0x00010B73, 0x00010B77}, -{0x00010B92, 0x00010B98}, {0x00010B9D, 0x00010BA8}, {0x00010BB0, 0x00010BFF}, {0x00010C49, 0x00010C7F}, -{0x00010CB3, 0x00010CBF}, {0x00010CF3, 0x00010CF9}, {0x00010D28, 0x00010D2F}, {0x00010D3A, 0x00010E5F}, -{0x00010E7F, 0x00010E7F}, {0x00010EAA, 0x00010EAA}, {0x00010EAE, 0x00010EAF}, {0x00010EB2, 0x00010EFC}, -{0x00010F28, 0x00010F2F}, {0x00010F5A, 0x00010F6F}, {0x00010F8A, 0x00010FAF}, {0x00010FCC, 0x00010FDF}, -{0x00010FF7, 0x00010FFF}, {0x0001104E, 0x00011051}, {0x00011076, 0x0001107E}, {0x000110BD, 0x000110BD}, -{0x000110C3, 0x000110CF}, {0x000110E9, 0x000110EF}, {0x000110FA, 0x000110FF}, {0x00011135, 0x00011135}, -{0x00011148, 0x0001114F}, {0x00011177, 0x0001117F}, {0x000111E0, 0x000111E0}, {0x000111F5, 0x000111FF}, -{0x00011212, 0x00011212}, {0x00011242, 0x0001127F}, {0x00011287, 0x00011287}, {0x00011289, 0x00011289}, -{0x0001128E, 0x0001128E}, {0x0001129E, 0x0001129E}, {0x000112AA, 0x000112AF}, {0x000112EB, 0x000112EF}, -{0x000112FA, 0x000112FF}, {0x00011304, 0x00011304}, {0x0001130D, 0x0001130E}, {0x00011311, 0x00011312}, -{0x00011329, 0x00011329}, {0x00011331, 0x00011331}, {0x00011334, 0x00011334}, {0x0001133A, 0x0001133A}, -{0x00011345, 0x00011346}, {0x00011349, 0x0001134A}, {0x0001134E, 0x0001134F}, {0x00011351, 0x00011356}, -{0x00011358, 0x0001135C}, {0x00011364, 0x00011365}, {0x0001136D, 0x0001136F}, {0x00011375, 0x000113FF}, -{0x0001145C, 0x0001145C}, {0x00011462, 0x0001147F}, {0x000114C8, 0x000114CF}, {0x000114DA, 0x0001157F}, -{0x000115B6, 0x000115B7}, {0x000115DE, 0x000115FF}, {0x00011645, 0x0001164F}, {0x0001165A, 0x0001165F}, -{0x0001166D, 0x0001167F}, {0x000116BA, 0x000116BF}, {0x000116CA, 0x000116FF}, {0x0001171B, 0x0001171C}, -{0x0001172C, 0x0001172F}, {0x00011747, 0x000117FF}, {0x0001183C, 0x0001189F}, {0x000118F3, 0x000118FE}, -{0x00011907, 0x00011908}, {0x0001190A, 0x0001190B}, {0x00011914, 0x00011914}, {0x00011917, 0x00011917}, -{0x00011936, 0x00011936}, {0x00011939, 0x0001193A}, {0x00011947, 0x0001194F}, {0x0001195A, 0x0001199F}, -{0x000119A8, 0x000119A9}, {0x000119D8, 0x000119D9}, {0x000119E5, 0x000119FF}, {0x00011A48, 0x00011A4F}, -{0x00011AA3, 0x00011AAF}, {0x00011AF9, 0x00011AFF}, {0x00011B0A, 0x00011BFF}, {0x00011C09, 0x00011C09}, -{0x00011C37, 0x00011C37}, {0x00011C46, 0x00011C4F}, {0x00011C6D, 0x00011C6F}, {0x00011C90, 0x00011C91}, -{0x00011CA8, 0x00011CA8}, {0x00011CB7, 0x00011CFF}, {0x00011D07, 0x00011D07}, {0x00011D0A, 0x00011D0A}, -{0x00011D37, 0x00011D39}, {0x00011D3B, 0x00011D3B}, {0x00011D3E, 0x00011D3E}, {0x00011D48, 0x00011D4F}, -{0x00011D5A, 0x00011D5F}, {0x00011D66, 0x00011D66}, {0x00011D69, 0x00011D69}, {0x00011D8F, 0x00011D8F}, -{0x00011D92, 0x00011D92}, {0x00011D99, 0x00011D9F}, {0x00011DAA, 0x00011EDF}, {0x00011EF9, 0x00011EFF}, -{0x00011F11, 0x00011F11}, {0x00011F3B, 0x00011F3D}, {0x00011F5A, 0x00011FAF}, {0x00011FB1, 0x00011FBF}, -{0x00011FF2, 0x00011FFE}, {0x0001239A, 0x000123FF}, {0x0001246F, 0x0001246F}, {0x00012475, 0x0001247F}, -{0x00012544, 0x00012F8F}, {0x00012FF3, 0x00012FFF}, {0x00013430, 0x0001343F}, {0x00013456, 0x000143FF}, -{0x00014647, 0x000167FF}, {0x00016A39, 0x00016A3F}, {0x00016A5F, 0x00016A5F}, {0x00016A6A, 0x00016A6D}, -{0x00016ABF, 0x00016ABF}, {0x00016ACA, 0x00016ACF}, {0x00016AEE, 0x00016AEF}, {0x00016AF6, 0x00016AFF}, -{0x00016B46, 0x00016B4F}, {0x00016B5A, 0x00016B5A}, {0x00016B62, 0x00016B62}, {0x00016B78, 0x00016B7C}, -{0x00016B90, 0x00016E3F}, {0x00016E9B, 0x00016EFF}, {0x00016F4B, 0x00016F4E}, {0x00016F88, 0x00016F8E}, -{0x00016FA0, 0x00016FDF}, {0x00016FE5, 0x00016FEF}, {0x00016FF2, 0x00016FFF}, {0x000187F8, 0x000187FF}, -{0x00018CD6, 0x00018CFF}, {0x00018D09, 0x0001AFEF}, {0x0001AFF4, 0x0001AFF4}, {0x0001AFFC, 0x0001AFFC}, -{0x0001AFFF, 0x0001AFFF}, {0x0001B123, 0x0001B131}, {0x0001B133, 0x0001B14F}, {0x0001B153, 0x0001B154}, -{0x0001B156, 0x0001B163}, {0x0001B168, 0x0001B16F}, {0x0001B2FC, 0x0001BBFF}, {0x0001BC6B, 0x0001BC6F}, -{0x0001BC7D, 0x0001BC7F}, {0x0001BC89, 0x0001BC8F}, {0x0001BC9A, 0x0001BC9B}, {0x0001BCA0, 0x0001CEFF}, -{0x0001CF2E, 0x0001CF2F}, {0x0001CF47, 0x0001CF4F}, {0x0001CFC4, 0x0001CFFF}, {0x0001D0F6, 0x0001D0FF}, -{0x0001D127, 0x0001D128}, {0x0001D173, 0x0001D17A}, {0x0001D1EB, 0x0001D1FF}, {0x0001D246, 0x0001D2BF}, -{0x0001D2D4, 0x0001D2DF}, {0x0001D2F4, 0x0001D2FF}, {0x0001D357, 0x0001D35F}, {0x0001D379, 0x0001D3FF}, -{0x0001D455, 0x0001D455}, {0x0001D49D, 0x0001D49D}, {0x0001D4A0, 0x0001D4A1}, {0x0001D4A3, 0x0001D4A4}, -{0x0001D4A7, 0x0001D4A8}, {0x0001D4AD, 0x0001D4AD}, {0x0001D4BA, 0x0001D4BA}, {0x0001D4BC, 0x0001D4BC}, -{0x0001D4C4, 0x0001D4C4}, {0x0001D506, 0x0001D506}, {0x0001D50B, 0x0001D50C}, {0x0001D515, 0x0001D515}, -{0x0001D51D, 0x0001D51D}, {0x0001D53A, 0x0001D53A}, {0x0001D53F, 0x0001D53F}, {0x0001D545, 0x0001D545}, -{0x0001D547, 0x0001D549}, {0x0001D551, 0x0001D551}, {0x0001D6A6, 0x0001D6A7}, {0x0001D7CC, 0x0001D7CD}, -{0x0001DA8C, 0x0001DA9A}, {0x0001DAA0, 0x0001DAA0}, {0x0001DAB0, 0x0001DEFF}, {0x0001DF1F, 0x0001DF24}, -{0x0001DF2B, 0x0001DFFF}, {0x0001E007, 0x0001E007}, {0x0001E019, 0x0001E01A}, {0x0001E022, 0x0001E022}, -{0x0001E025, 0x0001E025}, {0x0001E02B, 0x0001E02F}, {0x0001E06E, 0x0001E08E}, {0x0001E090, 0x0001E0FF}, -{0x0001E12D, 0x0001E12F}, {0x0001E13E, 0x0001E13F}, {0x0001E14A, 0x0001E14D}, {0x0001E150, 0x0001E28F}, -{0x0001E2AF, 0x0001E2BF}, {0x0001E2FA, 0x0001E2FE}, {0x0001E300, 0x0001E4CF}, {0x0001E4FA, 0x0001E7DF}, -{0x0001E7E7, 0x0001E7E7}, {0x0001E7EC, 0x0001E7EC}, {0x0001E7EF, 0x0001E7EF}, {0x0001E7FF, 0x0001E7FF}, -{0x0001E8C5, 0x0001E8C6}, {0x0001E8D7, 0x0001E8FF}, {0x0001E94C, 0x0001E94F}, {0x0001E95A, 0x0001E95D}, -{0x0001E960, 0x0001EC70}, {0x0001ECB5, 0x0001ED00}, {0x0001ED3E, 0x0001EDFF}, {0x0001EE04, 0x0001EE04}, -{0x0001EE20, 0x0001EE20}, {0x0001EE23, 0x0001EE23}, {0x0001EE25, 0x0001EE26}, {0x0001EE28, 0x0001EE28}, -{0x0001EE33, 0x0001EE33}, {0x0001EE38, 0x0001EE38}, {0x0001EE3A, 0x0001EE3A}, {0x0001EE3C, 0x0001EE41}, -{0x0001EE43, 0x0001EE46}, {0x0001EE48, 0x0001EE48}, {0x0001EE4A, 0x0001EE4A}, {0x0001EE4C, 0x0001EE4C}, -{0x0001EE50, 0x0001EE50}, {0x0001EE53, 0x0001EE53}, {0x0001EE55, 0x0001EE56}, {0x0001EE58, 0x0001EE58}, -{0x0001EE5A, 0x0001EE5A}, {0x0001EE5C, 0x0001EE5C}, {0x0001EE5E, 0x0001EE5E}, {0x0001EE60, 0x0001EE60}, -{0x0001EE63, 0x0001EE63}, {0x0001EE65, 0x0001EE66}, {0x0001EE6B, 0x0001EE6B}, {0x0001EE73, 0x0001EE73}, -{0x0001EE78, 0x0001EE78}, {0x0001EE7D, 0x0001EE7D}, {0x0001EE7F, 0x0001EE7F}, {0x0001EE8A, 0x0001EE8A}, -{0x0001EE9C, 0x0001EEA0}, {0x0001EEA4, 0x0001EEA4}, {0x0001EEAA, 0x0001EEAA}, {0x0001EEBC, 0x0001EEEF}, -{0x0001EEF2, 0x0001EFFF}, {0x0001F02C, 0x0001F02F}, {0x0001F094, 0x0001F09F}, {0x0001F0AF, 0x0001F0B0}, -{0x0001F0C0, 0x0001F0C0}, {0x0001F0D0, 0x0001F0D0}, {0x0001F0F6, 0x0001F0FF}, {0x0001F1AE, 0x0001F1E5}, -{0x0001F203, 0x0001F20F}, {0x0001F23C, 0x0001F23F}, {0x0001F249, 0x0001F24F}, {0x0001F252, 0x0001F25F}, -{0x0001F266, 0x0001F2FF}, {0x0001F6D8, 0x0001F6DB}, {0x0001F6ED, 0x0001F6EF}, {0x0001F6FD, 0x0001F6FF}, -{0x0001F777, 0x0001F77A}, {0x0001F7DA, 0x0001F7DF}, {0x0001F7EC, 0x0001F7EF}, {0x0001F7F1, 0x0001F7FF}, -{0x0001F80C, 0x0001F80F}, {0x0001F848, 0x0001F84F}, {0x0001F85A, 0x0001F85F}, {0x0001F888, 0x0001F88F}, -{0x0001F8AE, 0x0001F8AF}, {0x0001F8B2, 0x0001F8FF}, {0x0001FA54, 0x0001FA5F}, {0x0001FA6E, 0x0001FA6F}, -{0x0001FA7D, 0x0001FA7F}, {0x0001FA89, 0x0001FA8F}, {0x0001FABE, 0x0001FABE}, {0x0001FAC6, 0x0001FACD}, -{0x0001FADC, 0x0001FADF}, {0x0001FAE9, 0x0001FAEF}, {0x0001FAF9, 0x0001FAFF}, {0x0001FB93, 0x0001FB93}, -{0x0001FBCB, 0x0001FBEF}, {0x0001FBFA, 0x0001FFFF}, {0x0002A6E0, 0x0002A6FF}, {0x0002B73A, 0x0002B73F}, -{0x0002B81E, 0x0002B81F}, {0x0002CEA2, 0x0002CEAF}, {0x0002EBE1, 0x0002EBEF}, {0x0002EE5E, 0x0002F7FF}, -{0x0002FA1E, 0x0002FFFF}, {0x0003134B, 0x0003134F}, {0x000323B0, 0x000E00FF}, {0x000E01F0, 0x0010FFFF}, -}; - -const std::map unicode_map_lowercase = { -{0x00000041, 0x00000061}, {0x00000042, 0x00000062}, {0x00000043, 0x00000063}, {0x00000044, 0x00000064}, -{0x00000045, 0x00000065}, {0x00000046, 0x00000066}, {0x00000047, 0x00000067}, {0x00000048, 0x00000068}, -{0x00000049, 0x00000069}, {0x0000004A, 0x0000006A}, {0x0000004B, 0x0000006B}, {0x0000004C, 0x0000006C}, -{0x0000004D, 0x0000006D}, {0x0000004E, 0x0000006E}, {0x0000004F, 0x0000006F}, {0x00000050, 0x00000070}, -{0x00000051, 0x00000071}, {0x00000052, 0x00000072}, {0x00000053, 0x00000073}, {0x00000054, 0x00000074}, -{0x00000055, 0x00000075}, {0x00000056, 0x00000076}, {0x00000057, 0x00000077}, {0x00000058, 0x00000078}, -{0x00000059, 0x00000079}, {0x0000005A, 0x0000007A}, {0x000000C0, 0x000000E0}, {0x000000C1, 0x000000E1}, -{0x000000C2, 0x000000E2}, {0x000000C3, 0x000000E3}, {0x000000C4, 0x000000E4}, {0x000000C5, 0x000000E5}, -{0x000000C6, 0x000000E6}, {0x000000C7, 0x000000E7}, {0x000000C8, 0x000000E8}, {0x000000C9, 0x000000E9}, -{0x000000CA, 0x000000EA}, {0x000000CB, 0x000000EB}, {0x000000CC, 0x000000EC}, {0x000000CD, 0x000000ED}, -{0x000000CE, 0x000000EE}, {0x000000CF, 0x000000EF}, {0x000000D0, 0x000000F0}, {0x000000D1, 0x000000F1}, -{0x000000D2, 0x000000F2}, {0x000000D3, 0x000000F3}, {0x000000D4, 0x000000F4}, {0x000000D5, 0x000000F5}, -{0x000000D6, 0x000000F6}, {0x000000D8, 0x000000F8}, {0x000000D9, 0x000000F9}, {0x000000DA, 0x000000FA}, -{0x000000DB, 0x000000FB}, {0x000000DC, 0x000000FC}, {0x000000DD, 0x000000FD}, {0x000000DE, 0x000000FE}, -{0x00000100, 0x00000101}, {0x00000102, 0x00000103}, {0x00000104, 0x00000105}, {0x00000106, 0x00000107}, -{0x00000108, 0x00000109}, {0x0000010A, 0x0000010B}, {0x0000010C, 0x0000010D}, {0x0000010E, 0x0000010F}, -{0x00000110, 0x00000111}, {0x00000112, 0x00000113}, {0x00000114, 0x00000115}, {0x00000116, 0x00000117}, -{0x00000118, 0x00000119}, {0x0000011A, 0x0000011B}, {0x0000011C, 0x0000011D}, {0x0000011E, 0x0000011F}, -{0x00000120, 0x00000121}, {0x00000122, 0x00000123}, {0x00000124, 0x00000125}, {0x00000126, 0x00000127}, -{0x00000128, 0x00000129}, {0x0000012A, 0x0000012B}, {0x0000012C, 0x0000012D}, {0x0000012E, 0x0000012F}, -{0x00000130, 0x00000069}, {0x00000132, 0x00000133}, {0x00000134, 0x00000135}, {0x00000136, 0x00000137}, -{0x00000139, 0x0000013A}, {0x0000013B, 0x0000013C}, {0x0000013D, 0x0000013E}, {0x0000013F, 0x00000140}, -{0x00000141, 0x00000142}, {0x00000143, 0x00000144}, {0x00000145, 0x00000146}, {0x00000147, 0x00000148}, -{0x0000014A, 0x0000014B}, {0x0000014C, 0x0000014D}, {0x0000014E, 0x0000014F}, {0x00000150, 0x00000151}, -{0x00000152, 0x00000153}, {0x00000154, 0x00000155}, {0x00000156, 0x00000157}, {0x00000158, 0x00000159}, -{0x0000015A, 0x0000015B}, {0x0000015C, 0x0000015D}, {0x0000015E, 0x0000015F}, {0x00000160, 0x00000161}, -{0x00000162, 0x00000163}, {0x00000164, 0x00000165}, {0x00000166, 0x00000167}, {0x00000168, 0x00000169}, -{0x0000016A, 0x0000016B}, {0x0000016C, 0x0000016D}, {0x0000016E, 0x0000016F}, {0x00000170, 0x00000171}, -{0x00000172, 0x00000173}, {0x00000174, 0x00000175}, {0x00000176, 0x00000177}, {0x00000178, 0x000000FF}, -{0x00000179, 0x0000017A}, {0x0000017B, 0x0000017C}, {0x0000017D, 0x0000017E}, {0x00000181, 0x00000253}, -{0x00000182, 0x00000183}, {0x00000184, 0x00000185}, {0x00000186, 0x00000254}, {0x00000187, 0x00000188}, -{0x00000189, 0x00000256}, {0x0000018A, 0x00000257}, {0x0000018B, 0x0000018C}, {0x0000018E, 0x000001DD}, -{0x0000018F, 0x00000259}, {0x00000190, 0x0000025B}, {0x00000191, 0x00000192}, {0x00000193, 0x00000260}, -{0x00000194, 0x00000263}, {0x00000196, 0x00000269}, {0x00000197, 0x00000268}, {0x00000198, 0x00000199}, -{0x0000019C, 0x0000026F}, {0x0000019D, 0x00000272}, {0x0000019F, 0x00000275}, {0x000001A0, 0x000001A1}, -{0x000001A2, 0x000001A3}, {0x000001A4, 0x000001A5}, {0x000001A6, 0x00000280}, {0x000001A7, 0x000001A8}, -{0x000001A9, 0x00000283}, {0x000001AC, 0x000001AD}, {0x000001AE, 0x00000288}, {0x000001AF, 0x000001B0}, -{0x000001B1, 0x0000028A}, {0x000001B2, 0x0000028B}, {0x000001B3, 0x000001B4}, {0x000001B5, 0x000001B6}, -{0x000001B7, 0x00000292}, {0x000001B8, 0x000001B9}, {0x000001BC, 0x000001BD}, {0x000001C4, 0x000001C6}, -{0x000001C5, 0x000001C6}, {0x000001C7, 0x000001C9}, {0x000001C8, 0x000001C9}, {0x000001CA, 0x000001CC}, -{0x000001CB, 0x000001CC}, {0x000001CD, 0x000001CE}, {0x000001CF, 0x000001D0}, {0x000001D1, 0x000001D2}, -{0x000001D3, 0x000001D4}, {0x000001D5, 0x000001D6}, {0x000001D7, 0x000001D8}, {0x000001D9, 0x000001DA}, -{0x000001DB, 0x000001DC}, {0x000001DE, 0x000001DF}, {0x000001E0, 0x000001E1}, {0x000001E2, 0x000001E3}, -{0x000001E4, 0x000001E5}, {0x000001E6, 0x000001E7}, {0x000001E8, 0x000001E9}, {0x000001EA, 0x000001EB}, -{0x000001EC, 0x000001ED}, {0x000001EE, 0x000001EF}, {0x000001F1, 0x000001F3}, {0x000001F2, 0x000001F3}, -{0x000001F4, 0x000001F5}, {0x000001F6, 0x00000195}, {0x000001F7, 0x000001BF}, {0x000001F8, 0x000001F9}, -{0x000001FA, 0x000001FB}, {0x000001FC, 0x000001FD}, {0x000001FE, 0x000001FF}, {0x00000200, 0x00000201}, -{0x00000202, 0x00000203}, {0x00000204, 0x00000205}, {0x00000206, 0x00000207}, {0x00000208, 0x00000209}, -{0x0000020A, 0x0000020B}, {0x0000020C, 0x0000020D}, {0x0000020E, 0x0000020F}, {0x00000210, 0x00000211}, -{0x00000212, 0x00000213}, {0x00000214, 0x00000215}, {0x00000216, 0x00000217}, {0x00000218, 0x00000219}, -{0x0000021A, 0x0000021B}, {0x0000021C, 0x0000021D}, {0x0000021E, 0x0000021F}, {0x00000220, 0x0000019E}, -{0x00000222, 0x00000223}, {0x00000224, 0x00000225}, {0x00000226, 0x00000227}, {0x00000228, 0x00000229}, -{0x0000022A, 0x0000022B}, {0x0000022C, 0x0000022D}, {0x0000022E, 0x0000022F}, {0x00000230, 0x00000231}, -{0x00000232, 0x00000233}, {0x0000023A, 0x00002C65}, {0x0000023B, 0x0000023C}, {0x0000023D, 0x0000019A}, -{0x0000023E, 0x00002C66}, {0x00000241, 0x00000242}, {0x00000243, 0x00000180}, {0x00000244, 0x00000289}, -{0x00000245, 0x0000028C}, {0x00000246, 0x00000247}, {0x00000248, 0x00000249}, {0x0000024A, 0x0000024B}, -{0x0000024C, 0x0000024D}, {0x0000024E, 0x0000024F}, {0x00000370, 0x00000371}, {0x00000372, 0x00000373}, -{0x00000376, 0x00000377}, {0x0000037F, 0x000003F3}, {0x00000386, 0x000003AC}, {0x00000388, 0x000003AD}, -{0x00000389, 0x000003AE}, {0x0000038A, 0x000003AF}, {0x0000038C, 0x000003CC}, {0x0000038E, 0x000003CD}, -{0x0000038F, 0x000003CE}, {0x00000391, 0x000003B1}, {0x00000392, 0x000003B2}, {0x00000393, 0x000003B3}, -{0x00000394, 0x000003B4}, {0x00000395, 0x000003B5}, {0x00000396, 0x000003B6}, {0x00000397, 0x000003B7}, -{0x00000398, 0x000003B8}, {0x00000399, 0x000003B9}, {0x0000039A, 0x000003BA}, {0x0000039B, 0x000003BB}, -{0x0000039C, 0x000003BC}, {0x0000039D, 0x000003BD}, {0x0000039E, 0x000003BE}, {0x0000039F, 0x000003BF}, -{0x000003A0, 0x000003C0}, {0x000003A1, 0x000003C1}, {0x000003A3, 0x000003C3}, {0x000003A4, 0x000003C4}, -{0x000003A5, 0x000003C5}, {0x000003A6, 0x000003C6}, {0x000003A7, 0x000003C7}, {0x000003A8, 0x000003C8}, -{0x000003A9, 0x000003C9}, {0x000003AA, 0x000003CA}, {0x000003AB, 0x000003CB}, {0x000003CF, 0x000003D7}, -{0x000003D8, 0x000003D9}, {0x000003DA, 0x000003DB}, {0x000003DC, 0x000003DD}, {0x000003DE, 0x000003DF}, -{0x000003E0, 0x000003E1}, {0x000003E2, 0x000003E3}, {0x000003E4, 0x000003E5}, {0x000003E6, 0x000003E7}, -{0x000003E8, 0x000003E9}, {0x000003EA, 0x000003EB}, {0x000003EC, 0x000003ED}, {0x000003EE, 0x000003EF}, -{0x000003F4, 0x000003B8}, {0x000003F7, 0x000003F8}, {0x000003F9, 0x000003F2}, {0x000003FA, 0x000003FB}, -{0x000003FD, 0x0000037B}, {0x000003FE, 0x0000037C}, {0x000003FF, 0x0000037D}, {0x00000400, 0x00000450}, -{0x00000401, 0x00000451}, {0x00000402, 0x00000452}, {0x00000403, 0x00000453}, {0x00000404, 0x00000454}, -{0x00000405, 0x00000455}, {0x00000406, 0x00000456}, {0x00000407, 0x00000457}, {0x00000408, 0x00000458}, -{0x00000409, 0x00000459}, {0x0000040A, 0x0000045A}, {0x0000040B, 0x0000045B}, {0x0000040C, 0x0000045C}, -{0x0000040D, 0x0000045D}, {0x0000040E, 0x0000045E}, {0x0000040F, 0x0000045F}, {0x00000410, 0x00000430}, -{0x00000411, 0x00000431}, {0x00000412, 0x00000432}, {0x00000413, 0x00000433}, {0x00000414, 0x00000434}, -{0x00000415, 0x00000435}, {0x00000416, 0x00000436}, {0x00000417, 0x00000437}, {0x00000418, 0x00000438}, -{0x00000419, 0x00000439}, {0x0000041A, 0x0000043A}, {0x0000041B, 0x0000043B}, {0x0000041C, 0x0000043C}, -{0x0000041D, 0x0000043D}, {0x0000041E, 0x0000043E}, {0x0000041F, 0x0000043F}, {0x00000420, 0x00000440}, -{0x00000421, 0x00000441}, {0x00000422, 0x00000442}, {0x00000423, 0x00000443}, {0x00000424, 0x00000444}, -{0x00000425, 0x00000445}, {0x00000426, 0x00000446}, {0x00000427, 0x00000447}, {0x00000428, 0x00000448}, -{0x00000429, 0x00000449}, {0x0000042A, 0x0000044A}, {0x0000042B, 0x0000044B}, {0x0000042C, 0x0000044C}, -{0x0000042D, 0x0000044D}, {0x0000042E, 0x0000044E}, {0x0000042F, 0x0000044F}, {0x00000460, 0x00000461}, -{0x00000462, 0x00000463}, {0x00000464, 0x00000465}, {0x00000466, 0x00000467}, {0x00000468, 0x00000469}, -{0x0000046A, 0x0000046B}, {0x0000046C, 0x0000046D}, {0x0000046E, 0x0000046F}, {0x00000470, 0x00000471}, -{0x00000472, 0x00000473}, {0x00000474, 0x00000475}, {0x00000476, 0x00000477}, {0x00000478, 0x00000479}, -{0x0000047A, 0x0000047B}, {0x0000047C, 0x0000047D}, {0x0000047E, 0x0000047F}, {0x00000480, 0x00000481}, -{0x0000048A, 0x0000048B}, {0x0000048C, 0x0000048D}, {0x0000048E, 0x0000048F}, {0x00000490, 0x00000491}, -{0x00000492, 0x00000493}, {0x00000494, 0x00000495}, {0x00000496, 0x00000497}, {0x00000498, 0x00000499}, -{0x0000049A, 0x0000049B}, {0x0000049C, 0x0000049D}, {0x0000049E, 0x0000049F}, {0x000004A0, 0x000004A1}, -{0x000004A2, 0x000004A3}, {0x000004A4, 0x000004A5}, {0x000004A6, 0x000004A7}, {0x000004A8, 0x000004A9}, -{0x000004AA, 0x000004AB}, {0x000004AC, 0x000004AD}, {0x000004AE, 0x000004AF}, {0x000004B0, 0x000004B1}, -{0x000004B2, 0x000004B3}, {0x000004B4, 0x000004B5}, {0x000004B6, 0x000004B7}, {0x000004B8, 0x000004B9}, -{0x000004BA, 0x000004BB}, {0x000004BC, 0x000004BD}, {0x000004BE, 0x000004BF}, {0x000004C0, 0x000004CF}, -{0x000004C1, 0x000004C2}, {0x000004C3, 0x000004C4}, {0x000004C5, 0x000004C6}, {0x000004C7, 0x000004C8}, -{0x000004C9, 0x000004CA}, {0x000004CB, 0x000004CC}, {0x000004CD, 0x000004CE}, {0x000004D0, 0x000004D1}, -{0x000004D2, 0x000004D3}, {0x000004D4, 0x000004D5}, {0x000004D6, 0x000004D7}, {0x000004D8, 0x000004D9}, -{0x000004DA, 0x000004DB}, {0x000004DC, 0x000004DD}, {0x000004DE, 0x000004DF}, {0x000004E0, 0x000004E1}, -{0x000004E2, 0x000004E3}, {0x000004E4, 0x000004E5}, {0x000004E6, 0x000004E7}, {0x000004E8, 0x000004E9}, -{0x000004EA, 0x000004EB}, {0x000004EC, 0x000004ED}, {0x000004EE, 0x000004EF}, {0x000004F0, 0x000004F1}, -{0x000004F2, 0x000004F3}, {0x000004F4, 0x000004F5}, {0x000004F6, 0x000004F7}, {0x000004F8, 0x000004F9}, -{0x000004FA, 0x000004FB}, {0x000004FC, 0x000004FD}, {0x000004FE, 0x000004FF}, {0x00000500, 0x00000501}, -{0x00000502, 0x00000503}, {0x00000504, 0x00000505}, {0x00000506, 0x00000507}, {0x00000508, 0x00000509}, -{0x0000050A, 0x0000050B}, {0x0000050C, 0x0000050D}, {0x0000050E, 0x0000050F}, {0x00000510, 0x00000511}, -{0x00000512, 0x00000513}, {0x00000514, 0x00000515}, {0x00000516, 0x00000517}, {0x00000518, 0x00000519}, -{0x0000051A, 0x0000051B}, {0x0000051C, 0x0000051D}, {0x0000051E, 0x0000051F}, {0x00000520, 0x00000521}, -{0x00000522, 0x00000523}, {0x00000524, 0x00000525}, {0x00000526, 0x00000527}, {0x00000528, 0x00000529}, -{0x0000052A, 0x0000052B}, {0x0000052C, 0x0000052D}, {0x0000052E, 0x0000052F}, {0x00000531, 0x00000561}, -{0x00000532, 0x00000562}, {0x00000533, 0x00000563}, {0x00000534, 0x00000564}, {0x00000535, 0x00000565}, -{0x00000536, 0x00000566}, {0x00000537, 0x00000567}, {0x00000538, 0x00000568}, {0x00000539, 0x00000569}, -{0x0000053A, 0x0000056A}, {0x0000053B, 0x0000056B}, {0x0000053C, 0x0000056C}, {0x0000053D, 0x0000056D}, -{0x0000053E, 0x0000056E}, {0x0000053F, 0x0000056F}, {0x00000540, 0x00000570}, {0x00000541, 0x00000571}, -{0x00000542, 0x00000572}, {0x00000543, 0x00000573}, {0x00000544, 0x00000574}, {0x00000545, 0x00000575}, -{0x00000546, 0x00000576}, {0x00000547, 0x00000577}, {0x00000548, 0x00000578}, {0x00000549, 0x00000579}, -{0x0000054A, 0x0000057A}, {0x0000054B, 0x0000057B}, {0x0000054C, 0x0000057C}, {0x0000054D, 0x0000057D}, -{0x0000054E, 0x0000057E}, {0x0000054F, 0x0000057F}, {0x00000550, 0x00000580}, {0x00000551, 0x00000581}, -{0x00000552, 0x00000582}, {0x00000553, 0x00000583}, {0x00000554, 0x00000584}, {0x00000555, 0x00000585}, -{0x00000556, 0x00000586}, {0x000010A0, 0x00002D00}, {0x000010A1, 0x00002D01}, {0x000010A2, 0x00002D02}, -{0x000010A3, 0x00002D03}, {0x000010A4, 0x00002D04}, {0x000010A5, 0x00002D05}, {0x000010A6, 0x00002D06}, -{0x000010A7, 0x00002D07}, {0x000010A8, 0x00002D08}, {0x000010A9, 0x00002D09}, {0x000010AA, 0x00002D0A}, -{0x000010AB, 0x00002D0B}, {0x000010AC, 0x00002D0C}, {0x000010AD, 0x00002D0D}, {0x000010AE, 0x00002D0E}, -{0x000010AF, 0x00002D0F}, {0x000010B0, 0x00002D10}, {0x000010B1, 0x00002D11}, {0x000010B2, 0x00002D12}, -{0x000010B3, 0x00002D13}, {0x000010B4, 0x00002D14}, {0x000010B5, 0x00002D15}, {0x000010B6, 0x00002D16}, -{0x000010B7, 0x00002D17}, {0x000010B8, 0x00002D18}, {0x000010B9, 0x00002D19}, {0x000010BA, 0x00002D1A}, -{0x000010BB, 0x00002D1B}, {0x000010BC, 0x00002D1C}, {0x000010BD, 0x00002D1D}, {0x000010BE, 0x00002D1E}, -{0x000010BF, 0x00002D1F}, {0x000010C0, 0x00002D20}, {0x000010C1, 0x00002D21}, {0x000010C2, 0x00002D22}, -{0x000010C3, 0x00002D23}, {0x000010C4, 0x00002D24}, {0x000010C5, 0x00002D25}, {0x000010C7, 0x00002D27}, -{0x000010CD, 0x00002D2D}, {0x000013A0, 0x0000AB70}, {0x000013A1, 0x0000AB71}, {0x000013A2, 0x0000AB72}, -{0x000013A3, 0x0000AB73}, {0x000013A4, 0x0000AB74}, {0x000013A5, 0x0000AB75}, {0x000013A6, 0x0000AB76}, -{0x000013A7, 0x0000AB77}, {0x000013A8, 0x0000AB78}, {0x000013A9, 0x0000AB79}, {0x000013AA, 0x0000AB7A}, -{0x000013AB, 0x0000AB7B}, {0x000013AC, 0x0000AB7C}, {0x000013AD, 0x0000AB7D}, {0x000013AE, 0x0000AB7E}, -{0x000013AF, 0x0000AB7F}, {0x000013B0, 0x0000AB80}, {0x000013B1, 0x0000AB81}, {0x000013B2, 0x0000AB82}, -{0x000013B3, 0x0000AB83}, {0x000013B4, 0x0000AB84}, {0x000013B5, 0x0000AB85}, {0x000013B6, 0x0000AB86}, -{0x000013B7, 0x0000AB87}, {0x000013B8, 0x0000AB88}, {0x000013B9, 0x0000AB89}, {0x000013BA, 0x0000AB8A}, -{0x000013BB, 0x0000AB8B}, {0x000013BC, 0x0000AB8C}, {0x000013BD, 0x0000AB8D}, {0x000013BE, 0x0000AB8E}, -{0x000013BF, 0x0000AB8F}, {0x000013C0, 0x0000AB90}, {0x000013C1, 0x0000AB91}, {0x000013C2, 0x0000AB92}, -{0x000013C3, 0x0000AB93}, {0x000013C4, 0x0000AB94}, {0x000013C5, 0x0000AB95}, {0x000013C6, 0x0000AB96}, -{0x000013C7, 0x0000AB97}, {0x000013C8, 0x0000AB98}, {0x000013C9, 0x0000AB99}, {0x000013CA, 0x0000AB9A}, -{0x000013CB, 0x0000AB9B}, {0x000013CC, 0x0000AB9C}, {0x000013CD, 0x0000AB9D}, {0x000013CE, 0x0000AB9E}, -{0x000013CF, 0x0000AB9F}, {0x000013D0, 0x0000ABA0}, {0x000013D1, 0x0000ABA1}, {0x000013D2, 0x0000ABA2}, -{0x000013D3, 0x0000ABA3}, {0x000013D4, 0x0000ABA4}, {0x000013D5, 0x0000ABA5}, {0x000013D6, 0x0000ABA6}, -{0x000013D7, 0x0000ABA7}, {0x000013D8, 0x0000ABA8}, {0x000013D9, 0x0000ABA9}, {0x000013DA, 0x0000ABAA}, -{0x000013DB, 0x0000ABAB}, {0x000013DC, 0x0000ABAC}, {0x000013DD, 0x0000ABAD}, {0x000013DE, 0x0000ABAE}, -{0x000013DF, 0x0000ABAF}, {0x000013E0, 0x0000ABB0}, {0x000013E1, 0x0000ABB1}, {0x000013E2, 0x0000ABB2}, -{0x000013E3, 0x0000ABB3}, {0x000013E4, 0x0000ABB4}, {0x000013E5, 0x0000ABB5}, {0x000013E6, 0x0000ABB6}, -{0x000013E7, 0x0000ABB7}, {0x000013E8, 0x0000ABB8}, {0x000013E9, 0x0000ABB9}, {0x000013EA, 0x0000ABBA}, -{0x000013EB, 0x0000ABBB}, {0x000013EC, 0x0000ABBC}, {0x000013ED, 0x0000ABBD}, {0x000013EE, 0x0000ABBE}, -{0x000013EF, 0x0000ABBF}, {0x000013F0, 0x000013F8}, {0x000013F1, 0x000013F9}, {0x000013F2, 0x000013FA}, -{0x000013F3, 0x000013FB}, {0x000013F4, 0x000013FC}, {0x000013F5, 0x000013FD}, {0x00001C90, 0x000010D0}, -{0x00001C91, 0x000010D1}, {0x00001C92, 0x000010D2}, {0x00001C93, 0x000010D3}, {0x00001C94, 0x000010D4}, -{0x00001C95, 0x000010D5}, {0x00001C96, 0x000010D6}, {0x00001C97, 0x000010D7}, {0x00001C98, 0x000010D8}, -{0x00001C99, 0x000010D9}, {0x00001C9A, 0x000010DA}, {0x00001C9B, 0x000010DB}, {0x00001C9C, 0x000010DC}, -{0x00001C9D, 0x000010DD}, {0x00001C9E, 0x000010DE}, {0x00001C9F, 0x000010DF}, {0x00001CA0, 0x000010E0}, -{0x00001CA1, 0x000010E1}, {0x00001CA2, 0x000010E2}, {0x00001CA3, 0x000010E3}, {0x00001CA4, 0x000010E4}, -{0x00001CA5, 0x000010E5}, {0x00001CA6, 0x000010E6}, {0x00001CA7, 0x000010E7}, {0x00001CA8, 0x000010E8}, -{0x00001CA9, 0x000010E9}, {0x00001CAA, 0x000010EA}, {0x00001CAB, 0x000010EB}, {0x00001CAC, 0x000010EC}, -{0x00001CAD, 0x000010ED}, {0x00001CAE, 0x000010EE}, {0x00001CAF, 0x000010EF}, {0x00001CB0, 0x000010F0}, -{0x00001CB1, 0x000010F1}, {0x00001CB2, 0x000010F2}, {0x00001CB3, 0x000010F3}, {0x00001CB4, 0x000010F4}, -{0x00001CB5, 0x000010F5}, {0x00001CB6, 0x000010F6}, {0x00001CB7, 0x000010F7}, {0x00001CB8, 0x000010F8}, -{0x00001CB9, 0x000010F9}, {0x00001CBA, 0x000010FA}, {0x00001CBD, 0x000010FD}, {0x00001CBE, 0x000010FE}, -{0x00001CBF, 0x000010FF}, {0x00001E00, 0x00001E01}, {0x00001E02, 0x00001E03}, {0x00001E04, 0x00001E05}, -{0x00001E06, 0x00001E07}, {0x00001E08, 0x00001E09}, {0x00001E0A, 0x00001E0B}, {0x00001E0C, 0x00001E0D}, -{0x00001E0E, 0x00001E0F}, {0x00001E10, 0x00001E11}, {0x00001E12, 0x00001E13}, {0x00001E14, 0x00001E15}, -{0x00001E16, 0x00001E17}, {0x00001E18, 0x00001E19}, {0x00001E1A, 0x00001E1B}, {0x00001E1C, 0x00001E1D}, -{0x00001E1E, 0x00001E1F}, {0x00001E20, 0x00001E21}, {0x00001E22, 0x00001E23}, {0x00001E24, 0x00001E25}, -{0x00001E26, 0x00001E27}, {0x00001E28, 0x00001E29}, {0x00001E2A, 0x00001E2B}, {0x00001E2C, 0x00001E2D}, -{0x00001E2E, 0x00001E2F}, {0x00001E30, 0x00001E31}, {0x00001E32, 0x00001E33}, {0x00001E34, 0x00001E35}, -{0x00001E36, 0x00001E37}, {0x00001E38, 0x00001E39}, {0x00001E3A, 0x00001E3B}, {0x00001E3C, 0x00001E3D}, -{0x00001E3E, 0x00001E3F}, {0x00001E40, 0x00001E41}, {0x00001E42, 0x00001E43}, {0x00001E44, 0x00001E45}, -{0x00001E46, 0x00001E47}, {0x00001E48, 0x00001E49}, {0x00001E4A, 0x00001E4B}, {0x00001E4C, 0x00001E4D}, -{0x00001E4E, 0x00001E4F}, {0x00001E50, 0x00001E51}, {0x00001E52, 0x00001E53}, {0x00001E54, 0x00001E55}, -{0x00001E56, 0x00001E57}, {0x00001E58, 0x00001E59}, {0x00001E5A, 0x00001E5B}, {0x00001E5C, 0x00001E5D}, -{0x00001E5E, 0x00001E5F}, {0x00001E60, 0x00001E61}, {0x00001E62, 0x00001E63}, {0x00001E64, 0x00001E65}, -{0x00001E66, 0x00001E67}, {0x00001E68, 0x00001E69}, {0x00001E6A, 0x00001E6B}, {0x00001E6C, 0x00001E6D}, -{0x00001E6E, 0x00001E6F}, {0x00001E70, 0x00001E71}, {0x00001E72, 0x00001E73}, {0x00001E74, 0x00001E75}, -{0x00001E76, 0x00001E77}, {0x00001E78, 0x00001E79}, {0x00001E7A, 0x00001E7B}, {0x00001E7C, 0x00001E7D}, -{0x00001E7E, 0x00001E7F}, {0x00001E80, 0x00001E81}, {0x00001E82, 0x00001E83}, {0x00001E84, 0x00001E85}, -{0x00001E86, 0x00001E87}, {0x00001E88, 0x00001E89}, {0x00001E8A, 0x00001E8B}, {0x00001E8C, 0x00001E8D}, -{0x00001E8E, 0x00001E8F}, {0x00001E90, 0x00001E91}, {0x00001E92, 0x00001E93}, {0x00001E94, 0x00001E95}, -{0x00001E9E, 0x000000DF}, {0x00001EA0, 0x00001EA1}, {0x00001EA2, 0x00001EA3}, {0x00001EA4, 0x00001EA5}, -{0x00001EA6, 0x00001EA7}, {0x00001EA8, 0x00001EA9}, {0x00001EAA, 0x00001EAB}, {0x00001EAC, 0x00001EAD}, -{0x00001EAE, 0x00001EAF}, {0x00001EB0, 0x00001EB1}, {0x00001EB2, 0x00001EB3}, {0x00001EB4, 0x00001EB5}, -{0x00001EB6, 0x00001EB7}, {0x00001EB8, 0x00001EB9}, {0x00001EBA, 0x00001EBB}, {0x00001EBC, 0x00001EBD}, -{0x00001EBE, 0x00001EBF}, {0x00001EC0, 0x00001EC1}, {0x00001EC2, 0x00001EC3}, {0x00001EC4, 0x00001EC5}, -{0x00001EC6, 0x00001EC7}, {0x00001EC8, 0x00001EC9}, {0x00001ECA, 0x00001ECB}, {0x00001ECC, 0x00001ECD}, -{0x00001ECE, 0x00001ECF}, {0x00001ED0, 0x00001ED1}, {0x00001ED2, 0x00001ED3}, {0x00001ED4, 0x00001ED5}, -{0x00001ED6, 0x00001ED7}, {0x00001ED8, 0x00001ED9}, {0x00001EDA, 0x00001EDB}, {0x00001EDC, 0x00001EDD}, -{0x00001EDE, 0x00001EDF}, {0x00001EE0, 0x00001EE1}, {0x00001EE2, 0x00001EE3}, {0x00001EE4, 0x00001EE5}, -{0x00001EE6, 0x00001EE7}, {0x00001EE8, 0x00001EE9}, {0x00001EEA, 0x00001EEB}, {0x00001EEC, 0x00001EED}, -{0x00001EEE, 0x00001EEF}, {0x00001EF0, 0x00001EF1}, {0x00001EF2, 0x00001EF3}, {0x00001EF4, 0x00001EF5}, -{0x00001EF6, 0x00001EF7}, {0x00001EF8, 0x00001EF9}, {0x00001EFA, 0x00001EFB}, {0x00001EFC, 0x00001EFD}, -{0x00001EFE, 0x00001EFF}, {0x00001F08, 0x00001F00}, {0x00001F09, 0x00001F01}, {0x00001F0A, 0x00001F02}, -{0x00001F0B, 0x00001F03}, {0x00001F0C, 0x00001F04}, {0x00001F0D, 0x00001F05}, {0x00001F0E, 0x00001F06}, -{0x00001F0F, 0x00001F07}, {0x00001F18, 0x00001F10}, {0x00001F19, 0x00001F11}, {0x00001F1A, 0x00001F12}, -{0x00001F1B, 0x00001F13}, {0x00001F1C, 0x00001F14}, {0x00001F1D, 0x00001F15}, {0x00001F28, 0x00001F20}, -{0x00001F29, 0x00001F21}, {0x00001F2A, 0x00001F22}, {0x00001F2B, 0x00001F23}, {0x00001F2C, 0x00001F24}, -{0x00001F2D, 0x00001F25}, {0x00001F2E, 0x00001F26}, {0x00001F2F, 0x00001F27}, {0x00001F38, 0x00001F30}, -{0x00001F39, 0x00001F31}, {0x00001F3A, 0x00001F32}, {0x00001F3B, 0x00001F33}, {0x00001F3C, 0x00001F34}, -{0x00001F3D, 0x00001F35}, {0x00001F3E, 0x00001F36}, {0x00001F3F, 0x00001F37}, {0x00001F48, 0x00001F40}, -{0x00001F49, 0x00001F41}, {0x00001F4A, 0x00001F42}, {0x00001F4B, 0x00001F43}, {0x00001F4C, 0x00001F44}, -{0x00001F4D, 0x00001F45}, {0x00001F59, 0x00001F51}, {0x00001F5B, 0x00001F53}, {0x00001F5D, 0x00001F55}, -{0x00001F5F, 0x00001F57}, {0x00001F68, 0x00001F60}, {0x00001F69, 0x00001F61}, {0x00001F6A, 0x00001F62}, -{0x00001F6B, 0x00001F63}, {0x00001F6C, 0x00001F64}, {0x00001F6D, 0x00001F65}, {0x00001F6E, 0x00001F66}, -{0x00001F6F, 0x00001F67}, {0x00001F88, 0x00001F80}, {0x00001F89, 0x00001F81}, {0x00001F8A, 0x00001F82}, -{0x00001F8B, 0x00001F83}, {0x00001F8C, 0x00001F84}, {0x00001F8D, 0x00001F85}, {0x00001F8E, 0x00001F86}, -{0x00001F8F, 0x00001F87}, {0x00001F98, 0x00001F90}, {0x00001F99, 0x00001F91}, {0x00001F9A, 0x00001F92}, -{0x00001F9B, 0x00001F93}, {0x00001F9C, 0x00001F94}, {0x00001F9D, 0x00001F95}, {0x00001F9E, 0x00001F96}, -{0x00001F9F, 0x00001F97}, {0x00001FA8, 0x00001FA0}, {0x00001FA9, 0x00001FA1}, {0x00001FAA, 0x00001FA2}, -{0x00001FAB, 0x00001FA3}, {0x00001FAC, 0x00001FA4}, {0x00001FAD, 0x00001FA5}, {0x00001FAE, 0x00001FA6}, -{0x00001FAF, 0x00001FA7}, {0x00001FB8, 0x00001FB0}, {0x00001FB9, 0x00001FB1}, {0x00001FBA, 0x00001F70}, -{0x00001FBB, 0x00001F71}, {0x00001FBC, 0x00001FB3}, {0x00001FC8, 0x00001F72}, {0x00001FC9, 0x00001F73}, -{0x00001FCA, 0x00001F74}, {0x00001FCB, 0x00001F75}, {0x00001FCC, 0x00001FC3}, {0x00001FD8, 0x00001FD0}, -{0x00001FD9, 0x00001FD1}, {0x00001FDA, 0x00001F76}, {0x00001FDB, 0x00001F77}, {0x00001FE8, 0x00001FE0}, -{0x00001FE9, 0x00001FE1}, {0x00001FEA, 0x00001F7A}, {0x00001FEB, 0x00001F7B}, {0x00001FEC, 0x00001FE5}, -{0x00001FF8, 0x00001F78}, {0x00001FF9, 0x00001F79}, {0x00001FFA, 0x00001F7C}, {0x00001FFB, 0x00001F7D}, -{0x00001FFC, 0x00001FF3}, {0x00002126, 0x000003C9}, {0x0000212A, 0x0000006B}, {0x0000212B, 0x000000E5}, -{0x00002132, 0x0000214E}, {0x00002160, 0x00002170}, {0x00002161, 0x00002171}, {0x00002162, 0x00002172}, -{0x00002163, 0x00002173}, {0x00002164, 0x00002174}, {0x00002165, 0x00002175}, {0x00002166, 0x00002176}, -{0x00002167, 0x00002177}, {0x00002168, 0x00002178}, {0x00002169, 0x00002179}, {0x0000216A, 0x0000217A}, -{0x0000216B, 0x0000217B}, {0x0000216C, 0x0000217C}, {0x0000216D, 0x0000217D}, {0x0000216E, 0x0000217E}, -{0x0000216F, 0x0000217F}, {0x00002183, 0x00002184}, {0x000024B6, 0x000024D0}, {0x000024B7, 0x000024D1}, -{0x000024B8, 0x000024D2}, {0x000024B9, 0x000024D3}, {0x000024BA, 0x000024D4}, {0x000024BB, 0x000024D5}, -{0x000024BC, 0x000024D6}, {0x000024BD, 0x000024D7}, {0x000024BE, 0x000024D8}, {0x000024BF, 0x000024D9}, -{0x000024C0, 0x000024DA}, {0x000024C1, 0x000024DB}, {0x000024C2, 0x000024DC}, {0x000024C3, 0x000024DD}, -{0x000024C4, 0x000024DE}, {0x000024C5, 0x000024DF}, {0x000024C6, 0x000024E0}, {0x000024C7, 0x000024E1}, -{0x000024C8, 0x000024E2}, {0x000024C9, 0x000024E3}, {0x000024CA, 0x000024E4}, {0x000024CB, 0x000024E5}, -{0x000024CC, 0x000024E6}, {0x000024CD, 0x000024E7}, {0x000024CE, 0x000024E8}, {0x000024CF, 0x000024E9}, -{0x00002C00, 0x00002C30}, {0x00002C01, 0x00002C31}, {0x00002C02, 0x00002C32}, {0x00002C03, 0x00002C33}, -{0x00002C04, 0x00002C34}, {0x00002C05, 0x00002C35}, {0x00002C06, 0x00002C36}, {0x00002C07, 0x00002C37}, -{0x00002C08, 0x00002C38}, {0x00002C09, 0x00002C39}, {0x00002C0A, 0x00002C3A}, {0x00002C0B, 0x00002C3B}, -{0x00002C0C, 0x00002C3C}, {0x00002C0D, 0x00002C3D}, {0x00002C0E, 0x00002C3E}, {0x00002C0F, 0x00002C3F}, -{0x00002C10, 0x00002C40}, {0x00002C11, 0x00002C41}, {0x00002C12, 0x00002C42}, {0x00002C13, 0x00002C43}, -{0x00002C14, 0x00002C44}, {0x00002C15, 0x00002C45}, {0x00002C16, 0x00002C46}, {0x00002C17, 0x00002C47}, -{0x00002C18, 0x00002C48}, {0x00002C19, 0x00002C49}, {0x00002C1A, 0x00002C4A}, {0x00002C1B, 0x00002C4B}, -{0x00002C1C, 0x00002C4C}, {0x00002C1D, 0x00002C4D}, {0x00002C1E, 0x00002C4E}, {0x00002C1F, 0x00002C4F}, -{0x00002C20, 0x00002C50}, {0x00002C21, 0x00002C51}, {0x00002C22, 0x00002C52}, {0x00002C23, 0x00002C53}, -{0x00002C24, 0x00002C54}, {0x00002C25, 0x00002C55}, {0x00002C26, 0x00002C56}, {0x00002C27, 0x00002C57}, -{0x00002C28, 0x00002C58}, {0x00002C29, 0x00002C59}, {0x00002C2A, 0x00002C5A}, {0x00002C2B, 0x00002C5B}, -{0x00002C2C, 0x00002C5C}, {0x00002C2D, 0x00002C5D}, {0x00002C2E, 0x00002C5E}, {0x00002C60, 0x00002C61}, -{0x00002C62, 0x0000026B}, {0x00002C63, 0x00001D7D}, {0x00002C64, 0x0000027D}, {0x00002C67, 0x00002C68}, -{0x00002C69, 0x00002C6A}, {0x00002C6B, 0x00002C6C}, {0x00002C6D, 0x00000251}, {0x00002C6E, 0x00000271}, -{0x00002C6F, 0x00000250}, {0x00002C70, 0x00000252}, {0x00002C72, 0x00002C73}, {0x00002C75, 0x00002C76}, -{0x00002C7E, 0x0000023F}, {0x00002C7F, 0x00000240}, {0x00002C80, 0x00002C81}, {0x00002C82, 0x00002C83}, -{0x00002C84, 0x00002C85}, {0x00002C86, 0x00002C87}, {0x00002C88, 0x00002C89}, {0x00002C8A, 0x00002C8B}, -{0x00002C8C, 0x00002C8D}, {0x00002C8E, 0x00002C8F}, {0x00002C90, 0x00002C91}, {0x00002C92, 0x00002C93}, -{0x00002C94, 0x00002C95}, {0x00002C96, 0x00002C97}, {0x00002C98, 0x00002C99}, {0x00002C9A, 0x00002C9B}, -{0x00002C9C, 0x00002C9D}, {0x00002C9E, 0x00002C9F}, {0x00002CA0, 0x00002CA1}, {0x00002CA2, 0x00002CA3}, -{0x00002CA4, 0x00002CA5}, {0x00002CA6, 0x00002CA7}, {0x00002CA8, 0x00002CA9}, {0x00002CAA, 0x00002CAB}, -{0x00002CAC, 0x00002CAD}, {0x00002CAE, 0x00002CAF}, {0x00002CB0, 0x00002CB1}, {0x00002CB2, 0x00002CB3}, -{0x00002CB4, 0x00002CB5}, {0x00002CB6, 0x00002CB7}, {0x00002CB8, 0x00002CB9}, {0x00002CBA, 0x00002CBB}, -{0x00002CBC, 0x00002CBD}, {0x00002CBE, 0x00002CBF}, {0x00002CC0, 0x00002CC1}, {0x00002CC2, 0x00002CC3}, -{0x00002CC4, 0x00002CC5}, {0x00002CC6, 0x00002CC7}, {0x00002CC8, 0x00002CC9}, {0x00002CCA, 0x00002CCB}, -{0x00002CCC, 0x00002CCD}, {0x00002CCE, 0x00002CCF}, {0x00002CD0, 0x00002CD1}, {0x00002CD2, 0x00002CD3}, -{0x00002CD4, 0x00002CD5}, {0x00002CD6, 0x00002CD7}, {0x00002CD8, 0x00002CD9}, {0x00002CDA, 0x00002CDB}, -{0x00002CDC, 0x00002CDD}, {0x00002CDE, 0x00002CDF}, {0x00002CE0, 0x00002CE1}, {0x00002CE2, 0x00002CE3}, -{0x00002CEB, 0x00002CEC}, {0x00002CED, 0x00002CEE}, {0x00002CF2, 0x00002CF3}, {0x0000A640, 0x0000A641}, -{0x0000A642, 0x0000A643}, {0x0000A644, 0x0000A645}, {0x0000A646, 0x0000A647}, {0x0000A648, 0x0000A649}, -{0x0000A64A, 0x0000A64B}, {0x0000A64C, 0x0000A64D}, {0x0000A64E, 0x0000A64F}, {0x0000A650, 0x0000A651}, -{0x0000A652, 0x0000A653}, {0x0000A654, 0x0000A655}, {0x0000A656, 0x0000A657}, {0x0000A658, 0x0000A659}, -{0x0000A65A, 0x0000A65B}, {0x0000A65C, 0x0000A65D}, {0x0000A65E, 0x0000A65F}, {0x0000A660, 0x0000A661}, -{0x0000A662, 0x0000A663}, {0x0000A664, 0x0000A665}, {0x0000A666, 0x0000A667}, {0x0000A668, 0x0000A669}, -{0x0000A66A, 0x0000A66B}, {0x0000A66C, 0x0000A66D}, {0x0000A680, 0x0000A681}, {0x0000A682, 0x0000A683}, -{0x0000A684, 0x0000A685}, {0x0000A686, 0x0000A687}, {0x0000A688, 0x0000A689}, {0x0000A68A, 0x0000A68B}, -{0x0000A68C, 0x0000A68D}, {0x0000A68E, 0x0000A68F}, {0x0000A690, 0x0000A691}, {0x0000A692, 0x0000A693}, -{0x0000A694, 0x0000A695}, {0x0000A696, 0x0000A697}, {0x0000A698, 0x0000A699}, {0x0000A69A, 0x0000A69B}, -{0x0000A722, 0x0000A723}, {0x0000A724, 0x0000A725}, {0x0000A726, 0x0000A727}, {0x0000A728, 0x0000A729}, -{0x0000A72A, 0x0000A72B}, {0x0000A72C, 0x0000A72D}, {0x0000A72E, 0x0000A72F}, {0x0000A732, 0x0000A733}, -{0x0000A734, 0x0000A735}, {0x0000A736, 0x0000A737}, {0x0000A738, 0x0000A739}, {0x0000A73A, 0x0000A73B}, -{0x0000A73C, 0x0000A73D}, {0x0000A73E, 0x0000A73F}, {0x0000A740, 0x0000A741}, {0x0000A742, 0x0000A743}, -{0x0000A744, 0x0000A745}, {0x0000A746, 0x0000A747}, {0x0000A748, 0x0000A749}, {0x0000A74A, 0x0000A74B}, -{0x0000A74C, 0x0000A74D}, {0x0000A74E, 0x0000A74F}, {0x0000A750, 0x0000A751}, {0x0000A752, 0x0000A753}, -{0x0000A754, 0x0000A755}, {0x0000A756, 0x0000A757}, {0x0000A758, 0x0000A759}, {0x0000A75A, 0x0000A75B}, -{0x0000A75C, 0x0000A75D}, {0x0000A75E, 0x0000A75F}, {0x0000A760, 0x0000A761}, {0x0000A762, 0x0000A763}, -{0x0000A764, 0x0000A765}, {0x0000A766, 0x0000A767}, {0x0000A768, 0x0000A769}, {0x0000A76A, 0x0000A76B}, -{0x0000A76C, 0x0000A76D}, {0x0000A76E, 0x0000A76F}, {0x0000A779, 0x0000A77A}, {0x0000A77B, 0x0000A77C}, -{0x0000A77D, 0x00001D79}, {0x0000A77E, 0x0000A77F}, {0x0000A780, 0x0000A781}, {0x0000A782, 0x0000A783}, -{0x0000A784, 0x0000A785}, {0x0000A786, 0x0000A787}, {0x0000A78B, 0x0000A78C}, {0x0000A78D, 0x00000265}, -{0x0000A790, 0x0000A791}, {0x0000A792, 0x0000A793}, {0x0000A796, 0x0000A797}, {0x0000A798, 0x0000A799}, -{0x0000A79A, 0x0000A79B}, {0x0000A79C, 0x0000A79D}, {0x0000A79E, 0x0000A79F}, {0x0000A7A0, 0x0000A7A1}, -{0x0000A7A2, 0x0000A7A3}, {0x0000A7A4, 0x0000A7A5}, {0x0000A7A6, 0x0000A7A7}, {0x0000A7A8, 0x0000A7A9}, -{0x0000A7AA, 0x00000266}, {0x0000A7AB, 0x0000025C}, {0x0000A7AC, 0x00000261}, {0x0000A7AD, 0x0000026C}, -{0x0000A7AE, 0x0000026A}, {0x0000A7B0, 0x0000029E}, {0x0000A7B1, 0x00000287}, {0x0000A7B2, 0x0000029D}, -{0x0000A7B3, 0x0000AB53}, {0x0000A7B4, 0x0000A7B5}, {0x0000A7B6, 0x0000A7B7}, {0x0000A7B8, 0x0000A7B9}, -{0x0000A7BA, 0x0000A7BB}, {0x0000A7BC, 0x0000A7BD}, {0x0000A7BE, 0x0000A7BF}, {0x0000A7C2, 0x0000A7C3}, -{0x0000A7C4, 0x0000A794}, {0x0000A7C5, 0x00000282}, {0x0000A7C6, 0x00001D8E}, {0x0000A7C7, 0x0000A7C8}, -{0x0000A7C9, 0x0000A7CA}, {0x0000A7F5, 0x0000A7F6}, {0x0000FF21, 0x0000FF41}, {0x0000FF22, 0x0000FF42}, -{0x0000FF23, 0x0000FF43}, {0x0000FF24, 0x0000FF44}, {0x0000FF25, 0x0000FF45}, {0x0000FF26, 0x0000FF46}, -{0x0000FF27, 0x0000FF47}, {0x0000FF28, 0x0000FF48}, {0x0000FF29, 0x0000FF49}, {0x0000FF2A, 0x0000FF4A}, -{0x0000FF2B, 0x0000FF4B}, {0x0000FF2C, 0x0000FF4C}, {0x0000FF2D, 0x0000FF4D}, {0x0000FF2E, 0x0000FF4E}, -{0x0000FF2F, 0x0000FF4F}, {0x0000FF30, 0x0000FF50}, {0x0000FF31, 0x0000FF51}, {0x0000FF32, 0x0000FF52}, -{0x0000FF33, 0x0000FF53}, {0x0000FF34, 0x0000FF54}, {0x0000FF35, 0x0000FF55}, {0x0000FF36, 0x0000FF56}, -{0x0000FF37, 0x0000FF57}, {0x0000FF38, 0x0000FF58}, {0x0000FF39, 0x0000FF59}, {0x0000FF3A, 0x0000FF5A}, -{0x00010400, 0x00010428}, {0x00010401, 0x00010429}, {0x00010402, 0x0001042A}, {0x00010403, 0x0001042B}, -{0x00010404, 0x0001042C}, {0x00010405, 0x0001042D}, {0x00010406, 0x0001042E}, {0x00010407, 0x0001042F}, -{0x00010408, 0x00010430}, {0x00010409, 0x00010431}, {0x0001040A, 0x00010432}, {0x0001040B, 0x00010433}, -{0x0001040C, 0x00010434}, {0x0001040D, 0x00010435}, {0x0001040E, 0x00010436}, {0x0001040F, 0x00010437}, -{0x00010410, 0x00010438}, {0x00010411, 0x00010439}, {0x00010412, 0x0001043A}, {0x00010413, 0x0001043B}, -{0x00010414, 0x0001043C}, {0x00010415, 0x0001043D}, {0x00010416, 0x0001043E}, {0x00010417, 0x0001043F}, -{0x00010418, 0x00010440}, {0x00010419, 0x00010441}, {0x0001041A, 0x00010442}, {0x0001041B, 0x00010443}, -{0x0001041C, 0x00010444}, {0x0001041D, 0x00010445}, {0x0001041E, 0x00010446}, {0x0001041F, 0x00010447}, -{0x00010420, 0x00010448}, {0x00010421, 0x00010449}, {0x00010422, 0x0001044A}, {0x00010423, 0x0001044B}, -{0x00010424, 0x0001044C}, {0x00010425, 0x0001044D}, {0x00010426, 0x0001044E}, {0x00010427, 0x0001044F}, -{0x000104B0, 0x000104D8}, {0x000104B1, 0x000104D9}, {0x000104B2, 0x000104DA}, {0x000104B3, 0x000104DB}, -{0x000104B4, 0x000104DC}, {0x000104B5, 0x000104DD}, {0x000104B6, 0x000104DE}, {0x000104B7, 0x000104DF}, -{0x000104B8, 0x000104E0}, {0x000104B9, 0x000104E1}, {0x000104BA, 0x000104E2}, {0x000104BB, 0x000104E3}, -{0x000104BC, 0x000104E4}, {0x000104BD, 0x000104E5}, {0x000104BE, 0x000104E6}, {0x000104BF, 0x000104E7}, -{0x000104C0, 0x000104E8}, {0x000104C1, 0x000104E9}, {0x000104C2, 0x000104EA}, {0x000104C3, 0x000104EB}, -{0x000104C4, 0x000104EC}, {0x000104C5, 0x000104ED}, {0x000104C6, 0x000104EE}, {0x000104C7, 0x000104EF}, -{0x000104C8, 0x000104F0}, {0x000104C9, 0x000104F1}, {0x000104CA, 0x000104F2}, {0x000104CB, 0x000104F3}, -{0x000104CC, 0x000104F4}, {0x000104CD, 0x000104F5}, {0x000104CE, 0x000104F6}, {0x000104CF, 0x000104F7}, -{0x000104D0, 0x000104F8}, {0x000104D1, 0x000104F9}, {0x000104D2, 0x000104FA}, {0x000104D3, 0x000104FB}, -{0x00010C80, 0x00010CC0}, {0x00010C81, 0x00010CC1}, {0x00010C82, 0x00010CC2}, {0x00010C83, 0x00010CC3}, -{0x00010C84, 0x00010CC4}, {0x00010C85, 0x00010CC5}, {0x00010C86, 0x00010CC6}, {0x00010C87, 0x00010CC7}, -{0x00010C88, 0x00010CC8}, {0x00010C89, 0x00010CC9}, {0x00010C8A, 0x00010CCA}, {0x00010C8B, 0x00010CCB}, -{0x00010C8C, 0x00010CCC}, {0x00010C8D, 0x00010CCD}, {0x00010C8E, 0x00010CCE}, {0x00010C8F, 0x00010CCF}, -{0x00010C90, 0x00010CD0}, {0x00010C91, 0x00010CD1}, {0x00010C92, 0x00010CD2}, {0x00010C93, 0x00010CD3}, -{0x00010C94, 0x00010CD4}, {0x00010C95, 0x00010CD5}, {0x00010C96, 0x00010CD6}, {0x00010C97, 0x00010CD7}, -{0x00010C98, 0x00010CD8}, {0x00010C99, 0x00010CD9}, {0x00010C9A, 0x00010CDA}, {0x00010C9B, 0x00010CDB}, -{0x00010C9C, 0x00010CDC}, {0x00010C9D, 0x00010CDD}, {0x00010C9E, 0x00010CDE}, {0x00010C9F, 0x00010CDF}, -{0x00010CA0, 0x00010CE0}, {0x00010CA1, 0x00010CE1}, {0x00010CA2, 0x00010CE2}, {0x00010CA3, 0x00010CE3}, -{0x00010CA4, 0x00010CE4}, {0x00010CA5, 0x00010CE5}, {0x00010CA6, 0x00010CE6}, {0x00010CA7, 0x00010CE7}, -{0x00010CA8, 0x00010CE8}, {0x00010CA9, 0x00010CE9}, {0x00010CAA, 0x00010CEA}, {0x00010CAB, 0x00010CEB}, -{0x00010CAC, 0x00010CEC}, {0x00010CAD, 0x00010CED}, {0x00010CAE, 0x00010CEE}, {0x00010CAF, 0x00010CEF}, -{0x00010CB0, 0x00010CF0}, {0x00010CB1, 0x00010CF1}, {0x00010CB2, 0x00010CF2}, {0x000118A0, 0x000118C0}, -{0x000118A1, 0x000118C1}, {0x000118A2, 0x000118C2}, {0x000118A3, 0x000118C3}, {0x000118A4, 0x000118C4}, -{0x000118A5, 0x000118C5}, {0x000118A6, 0x000118C6}, {0x000118A7, 0x000118C7}, {0x000118A8, 0x000118C8}, -{0x000118A9, 0x000118C9}, {0x000118AA, 0x000118CA}, {0x000118AB, 0x000118CB}, {0x000118AC, 0x000118CC}, -{0x000118AD, 0x000118CD}, {0x000118AE, 0x000118CE}, {0x000118AF, 0x000118CF}, {0x000118B0, 0x000118D0}, -{0x000118B1, 0x000118D1}, {0x000118B2, 0x000118D2}, {0x000118B3, 0x000118D3}, {0x000118B4, 0x000118D4}, -{0x000118B5, 0x000118D5}, {0x000118B6, 0x000118D6}, {0x000118B7, 0x000118D7}, {0x000118B8, 0x000118D8}, -{0x000118B9, 0x000118D9}, {0x000118BA, 0x000118DA}, {0x000118BB, 0x000118DB}, {0x000118BC, 0x000118DC}, -{0x000118BD, 0x000118DD}, {0x000118BE, 0x000118DE}, {0x000118BF, 0x000118DF}, {0x00016E40, 0x00016E60}, -{0x00016E41, 0x00016E61}, {0x00016E42, 0x00016E62}, {0x00016E43, 0x00016E63}, {0x00016E44, 0x00016E64}, -{0x00016E45, 0x00016E65}, {0x00016E46, 0x00016E66}, {0x00016E47, 0x00016E67}, {0x00016E48, 0x00016E68}, -{0x00016E49, 0x00016E69}, {0x00016E4A, 0x00016E6A}, {0x00016E4B, 0x00016E6B}, {0x00016E4C, 0x00016E6C}, -{0x00016E4D, 0x00016E6D}, {0x00016E4E, 0x00016E6E}, {0x00016E4F, 0x00016E6F}, {0x00016E50, 0x00016E70}, -{0x00016E51, 0x00016E71}, {0x00016E52, 0x00016E72}, {0x00016E53, 0x00016E73}, {0x00016E54, 0x00016E74}, -{0x00016E55, 0x00016E75}, {0x00016E56, 0x00016E76}, {0x00016E57, 0x00016E77}, {0x00016E58, 0x00016E78}, -{0x00016E59, 0x00016E79}, {0x00016E5A, 0x00016E7A}, {0x00016E5B, 0x00016E7B}, {0x00016E5C, 0x00016E7C}, -{0x00016E5D, 0x00016E7D}, {0x00016E5E, 0x00016E7E}, {0x00016E5F, 0x00016E7F}, {0x0001E900, 0x0001E922}, -{0x0001E901, 0x0001E923}, {0x0001E902, 0x0001E924}, {0x0001E903, 0x0001E925}, {0x0001E904, 0x0001E926}, -{0x0001E905, 0x0001E927}, {0x0001E906, 0x0001E928}, {0x0001E907, 0x0001E929}, {0x0001E908, 0x0001E92A}, -{0x0001E909, 0x0001E92B}, {0x0001E90A, 0x0001E92C}, {0x0001E90B, 0x0001E92D}, {0x0001E90C, 0x0001E92E}, -{0x0001E90D, 0x0001E92F}, {0x0001E90E, 0x0001E930}, {0x0001E90F, 0x0001E931}, {0x0001E910, 0x0001E932}, -{0x0001E911, 0x0001E933}, {0x0001E912, 0x0001E934}, {0x0001E913, 0x0001E935}, {0x0001E914, 0x0001E936}, -{0x0001E915, 0x0001E937}, {0x0001E916, 0x0001E938}, {0x0001E917, 0x0001E939}, {0x0001E918, 0x0001E93A}, -{0x0001E919, 0x0001E93B}, {0x0001E91A, 0x0001E93C}, {0x0001E91B, 0x0001E93D}, {0x0001E91C, 0x0001E93E}, -{0x0001E91D, 0x0001E93F}, {0x0001E91E, 0x0001E940}, {0x0001E91F, 0x0001E941}, {0x0001E920, 0x0001E942}, -{0x0001E921, 0x0001E943}, -}; - -const std::map unicode_map_uppercase = { -{0x00000061, 0x00000041}, {0x00000062, 0x00000042}, {0x00000063, 0x00000043}, {0x00000064, 0x00000044}, -{0x00000065, 0x00000045}, {0x00000066, 0x00000046}, {0x00000067, 0x00000047}, {0x00000068, 0x00000048}, -{0x00000069, 0x00000049}, {0x0000006A, 0x0000004A}, {0x0000006B, 0x0000004B}, {0x0000006C, 0x0000004C}, -{0x0000006D, 0x0000004D}, {0x0000006E, 0x0000004E}, {0x0000006F, 0x0000004F}, {0x00000070, 0x00000050}, -{0x00000071, 0x00000051}, {0x00000072, 0x00000052}, {0x00000073, 0x00000053}, {0x00000074, 0x00000054}, -{0x00000075, 0x00000055}, {0x00000076, 0x00000056}, {0x00000077, 0x00000057}, {0x00000078, 0x00000058}, -{0x00000079, 0x00000059}, {0x0000007A, 0x0000005A}, {0x000000B5, 0x0000039C}, {0x000000DF, 0x00000053}, -{0x000000E0, 0x000000C0}, {0x000000E1, 0x000000C1}, {0x000000E2, 0x000000C2}, {0x000000E3, 0x000000C3}, -{0x000000E4, 0x000000C4}, {0x000000E5, 0x000000C5}, {0x000000E6, 0x000000C6}, {0x000000E7, 0x000000C7}, -{0x000000E8, 0x000000C8}, {0x000000E9, 0x000000C9}, {0x000000EA, 0x000000CA}, {0x000000EB, 0x000000CB}, -{0x000000EC, 0x000000CC}, {0x000000ED, 0x000000CD}, {0x000000EE, 0x000000CE}, {0x000000EF, 0x000000CF}, -{0x000000F0, 0x000000D0}, {0x000000F1, 0x000000D1}, {0x000000F2, 0x000000D2}, {0x000000F3, 0x000000D3}, -{0x000000F4, 0x000000D4}, {0x000000F5, 0x000000D5}, {0x000000F6, 0x000000D6}, {0x000000F8, 0x000000D8}, -{0x000000F9, 0x000000D9}, {0x000000FA, 0x000000DA}, {0x000000FB, 0x000000DB}, {0x000000FC, 0x000000DC}, -{0x000000FD, 0x000000DD}, {0x000000FE, 0x000000DE}, {0x000000FF, 0x00000178}, {0x00000101, 0x00000100}, -{0x00000103, 0x00000102}, {0x00000105, 0x00000104}, {0x00000107, 0x00000106}, {0x00000109, 0x00000108}, -{0x0000010B, 0x0000010A}, {0x0000010D, 0x0000010C}, {0x0000010F, 0x0000010E}, {0x00000111, 0x00000110}, -{0x00000113, 0x00000112}, {0x00000115, 0x00000114}, {0x00000117, 0x00000116}, {0x00000119, 0x00000118}, -{0x0000011B, 0x0000011A}, {0x0000011D, 0x0000011C}, {0x0000011F, 0x0000011E}, {0x00000121, 0x00000120}, -{0x00000123, 0x00000122}, {0x00000125, 0x00000124}, {0x00000127, 0x00000126}, {0x00000129, 0x00000128}, -{0x0000012B, 0x0000012A}, {0x0000012D, 0x0000012C}, {0x0000012F, 0x0000012E}, {0x00000131, 0x00000049}, -{0x00000133, 0x00000132}, {0x00000135, 0x00000134}, {0x00000137, 0x00000136}, {0x0000013A, 0x00000139}, -{0x0000013C, 0x0000013B}, {0x0000013E, 0x0000013D}, {0x00000140, 0x0000013F}, {0x00000142, 0x00000141}, -{0x00000144, 0x00000143}, {0x00000146, 0x00000145}, {0x00000148, 0x00000147}, {0x00000149, 0x000002BC}, -{0x0000014B, 0x0000014A}, {0x0000014D, 0x0000014C}, {0x0000014F, 0x0000014E}, {0x00000151, 0x00000150}, -{0x00000153, 0x00000152}, {0x00000155, 0x00000154}, {0x00000157, 0x00000156}, {0x00000159, 0x00000158}, -{0x0000015B, 0x0000015A}, {0x0000015D, 0x0000015C}, {0x0000015F, 0x0000015E}, {0x00000161, 0x00000160}, -{0x00000163, 0x00000162}, {0x00000165, 0x00000164}, {0x00000167, 0x00000166}, {0x00000169, 0x00000168}, -{0x0000016B, 0x0000016A}, {0x0000016D, 0x0000016C}, {0x0000016F, 0x0000016E}, {0x00000171, 0x00000170}, -{0x00000173, 0x00000172}, {0x00000175, 0x00000174}, {0x00000177, 0x00000176}, {0x0000017A, 0x00000179}, -{0x0000017C, 0x0000017B}, {0x0000017E, 0x0000017D}, {0x0000017F, 0x00000053}, {0x00000180, 0x00000243}, -{0x00000183, 0x00000182}, {0x00000185, 0x00000184}, {0x00000188, 0x00000187}, {0x0000018C, 0x0000018B}, -{0x00000192, 0x00000191}, {0x00000195, 0x000001F6}, {0x00000199, 0x00000198}, {0x0000019A, 0x0000023D}, -{0x0000019E, 0x00000220}, {0x000001A1, 0x000001A0}, {0x000001A3, 0x000001A2}, {0x000001A5, 0x000001A4}, -{0x000001A8, 0x000001A7}, {0x000001AD, 0x000001AC}, {0x000001B0, 0x000001AF}, {0x000001B4, 0x000001B3}, -{0x000001B6, 0x000001B5}, {0x000001B9, 0x000001B8}, {0x000001BD, 0x000001BC}, {0x000001BF, 0x000001F7}, -{0x000001C5, 0x000001C4}, {0x000001C6, 0x000001C4}, {0x000001C8, 0x000001C7}, {0x000001C9, 0x000001C7}, -{0x000001CB, 0x000001CA}, {0x000001CC, 0x000001CA}, {0x000001CE, 0x000001CD}, {0x000001D0, 0x000001CF}, -{0x000001D2, 0x000001D1}, {0x000001D4, 0x000001D3}, {0x000001D6, 0x000001D5}, {0x000001D8, 0x000001D7}, -{0x000001DA, 0x000001D9}, {0x000001DC, 0x000001DB}, {0x000001DD, 0x0000018E}, {0x000001DF, 0x000001DE}, -{0x000001E1, 0x000001E0}, {0x000001E3, 0x000001E2}, {0x000001E5, 0x000001E4}, {0x000001E7, 0x000001E6}, -{0x000001E9, 0x000001E8}, {0x000001EB, 0x000001EA}, {0x000001ED, 0x000001EC}, {0x000001EF, 0x000001EE}, -{0x000001F0, 0x0000004A}, {0x000001F2, 0x000001F1}, {0x000001F3, 0x000001F1}, {0x000001F5, 0x000001F4}, -{0x000001F9, 0x000001F8}, {0x000001FB, 0x000001FA}, {0x000001FD, 0x000001FC}, {0x000001FF, 0x000001FE}, -{0x00000201, 0x00000200}, {0x00000203, 0x00000202}, {0x00000205, 0x00000204}, {0x00000207, 0x00000206}, -{0x00000209, 0x00000208}, {0x0000020B, 0x0000020A}, {0x0000020D, 0x0000020C}, {0x0000020F, 0x0000020E}, -{0x00000211, 0x00000210}, {0x00000213, 0x00000212}, {0x00000215, 0x00000214}, {0x00000217, 0x00000216}, -{0x00000219, 0x00000218}, {0x0000021B, 0x0000021A}, {0x0000021D, 0x0000021C}, {0x0000021F, 0x0000021E}, -{0x00000223, 0x00000222}, {0x00000225, 0x00000224}, {0x00000227, 0x00000226}, {0x00000229, 0x00000228}, -{0x0000022B, 0x0000022A}, {0x0000022D, 0x0000022C}, {0x0000022F, 0x0000022E}, {0x00000231, 0x00000230}, -{0x00000233, 0x00000232}, {0x0000023C, 0x0000023B}, {0x0000023F, 0x00002C7E}, {0x00000240, 0x00002C7F}, -{0x00000242, 0x00000241}, {0x00000247, 0x00000246}, {0x00000249, 0x00000248}, {0x0000024B, 0x0000024A}, -{0x0000024D, 0x0000024C}, {0x0000024F, 0x0000024E}, {0x00000250, 0x00002C6F}, {0x00000251, 0x00002C6D}, -{0x00000252, 0x00002C70}, {0x00000253, 0x00000181}, {0x00000254, 0x00000186}, {0x00000256, 0x00000189}, -{0x00000257, 0x0000018A}, {0x00000259, 0x0000018F}, {0x0000025B, 0x00000190}, {0x0000025C, 0x0000A7AB}, -{0x00000260, 0x00000193}, {0x00000261, 0x0000A7AC}, {0x00000263, 0x00000194}, {0x00000265, 0x0000A78D}, -{0x00000266, 0x0000A7AA}, {0x00000268, 0x00000197}, {0x00000269, 0x00000196}, {0x0000026A, 0x0000A7AE}, -{0x0000026B, 0x00002C62}, {0x0000026C, 0x0000A7AD}, {0x0000026F, 0x0000019C}, {0x00000271, 0x00002C6E}, -{0x00000272, 0x0000019D}, {0x00000275, 0x0000019F}, {0x0000027D, 0x00002C64}, {0x00000280, 0x000001A6}, -{0x00000282, 0x0000A7C5}, {0x00000283, 0x000001A9}, {0x00000287, 0x0000A7B1}, {0x00000288, 0x000001AE}, -{0x00000289, 0x00000244}, {0x0000028A, 0x000001B1}, {0x0000028B, 0x000001B2}, {0x0000028C, 0x00000245}, -{0x00000292, 0x000001B7}, {0x0000029D, 0x0000A7B2}, {0x0000029E, 0x0000A7B0}, {0x00000345, 0x00000399}, -{0x00000371, 0x00000370}, {0x00000373, 0x00000372}, {0x00000377, 0x00000376}, {0x0000037B, 0x000003FD}, -{0x0000037C, 0x000003FE}, {0x0000037D, 0x000003FF}, {0x00000390, 0x00000399}, {0x000003AC, 0x00000386}, -{0x000003AD, 0x00000388}, {0x000003AE, 0x00000389}, {0x000003AF, 0x0000038A}, {0x000003B0, 0x000003A5}, -{0x000003B1, 0x00000391}, {0x000003B2, 0x00000392}, {0x000003B3, 0x00000393}, {0x000003B4, 0x00000394}, -{0x000003B5, 0x00000395}, {0x000003B6, 0x00000396}, {0x000003B7, 0x00000397}, {0x000003B8, 0x00000398}, -{0x000003B9, 0x00000399}, {0x000003BA, 0x0000039A}, {0x000003BB, 0x0000039B}, {0x000003BC, 0x0000039C}, -{0x000003BD, 0x0000039D}, {0x000003BE, 0x0000039E}, {0x000003BF, 0x0000039F}, {0x000003C0, 0x000003A0}, -{0x000003C1, 0x000003A1}, {0x000003C2, 0x000003A3}, {0x000003C3, 0x000003A3}, {0x000003C4, 0x000003A4}, -{0x000003C5, 0x000003A5}, {0x000003C6, 0x000003A6}, {0x000003C7, 0x000003A7}, {0x000003C8, 0x000003A8}, -{0x000003C9, 0x000003A9}, {0x000003CA, 0x000003AA}, {0x000003CB, 0x000003AB}, {0x000003CC, 0x0000038C}, -{0x000003CD, 0x0000038E}, {0x000003CE, 0x0000038F}, {0x000003D0, 0x00000392}, {0x000003D1, 0x00000398}, -{0x000003D5, 0x000003A6}, {0x000003D6, 0x000003A0}, {0x000003D7, 0x000003CF}, {0x000003D9, 0x000003D8}, -{0x000003DB, 0x000003DA}, {0x000003DD, 0x000003DC}, {0x000003DF, 0x000003DE}, {0x000003E1, 0x000003E0}, -{0x000003E3, 0x000003E2}, {0x000003E5, 0x000003E4}, {0x000003E7, 0x000003E6}, {0x000003E9, 0x000003E8}, -{0x000003EB, 0x000003EA}, {0x000003ED, 0x000003EC}, {0x000003EF, 0x000003EE}, {0x000003F0, 0x0000039A}, -{0x000003F1, 0x000003A1}, {0x000003F2, 0x000003F9}, {0x000003F3, 0x0000037F}, {0x000003F5, 0x00000395}, -{0x000003F8, 0x000003F7}, {0x000003FB, 0x000003FA}, {0x00000430, 0x00000410}, {0x00000431, 0x00000411}, -{0x00000432, 0x00000412}, {0x00000433, 0x00000413}, {0x00000434, 0x00000414}, {0x00000435, 0x00000415}, -{0x00000436, 0x00000416}, {0x00000437, 0x00000417}, {0x00000438, 0x00000418}, {0x00000439, 0x00000419}, -{0x0000043A, 0x0000041A}, {0x0000043B, 0x0000041B}, {0x0000043C, 0x0000041C}, {0x0000043D, 0x0000041D}, -{0x0000043E, 0x0000041E}, {0x0000043F, 0x0000041F}, {0x00000440, 0x00000420}, {0x00000441, 0x00000421}, -{0x00000442, 0x00000422}, {0x00000443, 0x00000423}, {0x00000444, 0x00000424}, {0x00000445, 0x00000425}, -{0x00000446, 0x00000426}, {0x00000447, 0x00000427}, {0x00000448, 0x00000428}, {0x00000449, 0x00000429}, -{0x0000044A, 0x0000042A}, {0x0000044B, 0x0000042B}, {0x0000044C, 0x0000042C}, {0x0000044D, 0x0000042D}, -{0x0000044E, 0x0000042E}, {0x0000044F, 0x0000042F}, {0x00000450, 0x00000400}, {0x00000451, 0x00000401}, -{0x00000452, 0x00000402}, {0x00000453, 0x00000403}, {0x00000454, 0x00000404}, {0x00000455, 0x00000405}, -{0x00000456, 0x00000406}, {0x00000457, 0x00000407}, {0x00000458, 0x00000408}, {0x00000459, 0x00000409}, -{0x0000045A, 0x0000040A}, {0x0000045B, 0x0000040B}, {0x0000045C, 0x0000040C}, {0x0000045D, 0x0000040D}, -{0x0000045E, 0x0000040E}, {0x0000045F, 0x0000040F}, {0x00000461, 0x00000460}, {0x00000463, 0x00000462}, -{0x00000465, 0x00000464}, {0x00000467, 0x00000466}, {0x00000469, 0x00000468}, {0x0000046B, 0x0000046A}, -{0x0000046D, 0x0000046C}, {0x0000046F, 0x0000046E}, {0x00000471, 0x00000470}, {0x00000473, 0x00000472}, -{0x00000475, 0x00000474}, {0x00000477, 0x00000476}, {0x00000479, 0x00000478}, {0x0000047B, 0x0000047A}, -{0x0000047D, 0x0000047C}, {0x0000047F, 0x0000047E}, {0x00000481, 0x00000480}, {0x0000048B, 0x0000048A}, -{0x0000048D, 0x0000048C}, {0x0000048F, 0x0000048E}, {0x00000491, 0x00000490}, {0x00000493, 0x00000492}, -{0x00000495, 0x00000494}, {0x00000497, 0x00000496}, {0x00000499, 0x00000498}, {0x0000049B, 0x0000049A}, -{0x0000049D, 0x0000049C}, {0x0000049F, 0x0000049E}, {0x000004A1, 0x000004A0}, {0x000004A3, 0x000004A2}, -{0x000004A5, 0x000004A4}, {0x000004A7, 0x000004A6}, {0x000004A9, 0x000004A8}, {0x000004AB, 0x000004AA}, -{0x000004AD, 0x000004AC}, {0x000004AF, 0x000004AE}, {0x000004B1, 0x000004B0}, {0x000004B3, 0x000004B2}, -{0x000004B5, 0x000004B4}, {0x000004B7, 0x000004B6}, {0x000004B9, 0x000004B8}, {0x000004BB, 0x000004BA}, -{0x000004BD, 0x000004BC}, {0x000004BF, 0x000004BE}, {0x000004C2, 0x000004C1}, {0x000004C4, 0x000004C3}, -{0x000004C6, 0x000004C5}, {0x000004C8, 0x000004C7}, {0x000004CA, 0x000004C9}, {0x000004CC, 0x000004CB}, -{0x000004CE, 0x000004CD}, {0x000004CF, 0x000004C0}, {0x000004D1, 0x000004D0}, {0x000004D3, 0x000004D2}, -{0x000004D5, 0x000004D4}, {0x000004D7, 0x000004D6}, {0x000004D9, 0x000004D8}, {0x000004DB, 0x000004DA}, -{0x000004DD, 0x000004DC}, {0x000004DF, 0x000004DE}, {0x000004E1, 0x000004E0}, {0x000004E3, 0x000004E2}, -{0x000004E5, 0x000004E4}, {0x000004E7, 0x000004E6}, {0x000004E9, 0x000004E8}, {0x000004EB, 0x000004EA}, -{0x000004ED, 0x000004EC}, {0x000004EF, 0x000004EE}, {0x000004F1, 0x000004F0}, {0x000004F3, 0x000004F2}, -{0x000004F5, 0x000004F4}, {0x000004F7, 0x000004F6}, {0x000004F9, 0x000004F8}, {0x000004FB, 0x000004FA}, -{0x000004FD, 0x000004FC}, {0x000004FF, 0x000004FE}, {0x00000501, 0x00000500}, {0x00000503, 0x00000502}, -{0x00000505, 0x00000504}, {0x00000507, 0x00000506}, {0x00000509, 0x00000508}, {0x0000050B, 0x0000050A}, -{0x0000050D, 0x0000050C}, {0x0000050F, 0x0000050E}, {0x00000511, 0x00000510}, {0x00000513, 0x00000512}, -{0x00000515, 0x00000514}, {0x00000517, 0x00000516}, {0x00000519, 0x00000518}, {0x0000051B, 0x0000051A}, -{0x0000051D, 0x0000051C}, {0x0000051F, 0x0000051E}, {0x00000521, 0x00000520}, {0x00000523, 0x00000522}, -{0x00000525, 0x00000524}, {0x00000527, 0x00000526}, {0x00000529, 0x00000528}, {0x0000052B, 0x0000052A}, -{0x0000052D, 0x0000052C}, {0x0000052F, 0x0000052E}, {0x00000561, 0x00000531}, {0x00000562, 0x00000532}, -{0x00000563, 0x00000533}, {0x00000564, 0x00000534}, {0x00000565, 0x00000535}, {0x00000566, 0x00000536}, -{0x00000567, 0x00000537}, {0x00000568, 0x00000538}, {0x00000569, 0x00000539}, {0x0000056A, 0x0000053A}, -{0x0000056B, 0x0000053B}, {0x0000056C, 0x0000053C}, {0x0000056D, 0x0000053D}, {0x0000056E, 0x0000053E}, -{0x0000056F, 0x0000053F}, {0x00000570, 0x00000540}, {0x00000571, 0x00000541}, {0x00000572, 0x00000542}, -{0x00000573, 0x00000543}, {0x00000574, 0x00000544}, {0x00000575, 0x00000545}, {0x00000576, 0x00000546}, -{0x00000577, 0x00000547}, {0x00000578, 0x00000548}, {0x00000579, 0x00000549}, {0x0000057A, 0x0000054A}, -{0x0000057B, 0x0000054B}, {0x0000057C, 0x0000054C}, {0x0000057D, 0x0000054D}, {0x0000057E, 0x0000054E}, -{0x0000057F, 0x0000054F}, {0x00000580, 0x00000550}, {0x00000581, 0x00000551}, {0x00000582, 0x00000552}, -{0x00000583, 0x00000553}, {0x00000584, 0x00000554}, {0x00000585, 0x00000555}, {0x00000586, 0x00000556}, -{0x00000587, 0x00000535}, {0x000010D0, 0x00001C90}, {0x000010D1, 0x00001C91}, {0x000010D2, 0x00001C92}, -{0x000010D3, 0x00001C93}, {0x000010D4, 0x00001C94}, {0x000010D5, 0x00001C95}, {0x000010D6, 0x00001C96}, -{0x000010D7, 0x00001C97}, {0x000010D8, 0x00001C98}, {0x000010D9, 0x00001C99}, {0x000010DA, 0x00001C9A}, -{0x000010DB, 0x00001C9B}, {0x000010DC, 0x00001C9C}, {0x000010DD, 0x00001C9D}, {0x000010DE, 0x00001C9E}, -{0x000010DF, 0x00001C9F}, {0x000010E0, 0x00001CA0}, {0x000010E1, 0x00001CA1}, {0x000010E2, 0x00001CA2}, -{0x000010E3, 0x00001CA3}, {0x000010E4, 0x00001CA4}, {0x000010E5, 0x00001CA5}, {0x000010E6, 0x00001CA6}, -{0x000010E7, 0x00001CA7}, {0x000010E8, 0x00001CA8}, {0x000010E9, 0x00001CA9}, {0x000010EA, 0x00001CAA}, -{0x000010EB, 0x00001CAB}, {0x000010EC, 0x00001CAC}, {0x000010ED, 0x00001CAD}, {0x000010EE, 0x00001CAE}, -{0x000010EF, 0x00001CAF}, {0x000010F0, 0x00001CB0}, {0x000010F1, 0x00001CB1}, {0x000010F2, 0x00001CB2}, -{0x000010F3, 0x00001CB3}, {0x000010F4, 0x00001CB4}, {0x000010F5, 0x00001CB5}, {0x000010F6, 0x00001CB6}, -{0x000010F7, 0x00001CB7}, {0x000010F8, 0x00001CB8}, {0x000010F9, 0x00001CB9}, {0x000010FA, 0x00001CBA}, -{0x000010FD, 0x00001CBD}, {0x000010FE, 0x00001CBE}, {0x000010FF, 0x00001CBF}, {0x000013F8, 0x000013F0}, -{0x000013F9, 0x000013F1}, {0x000013FA, 0x000013F2}, {0x000013FB, 0x000013F3}, {0x000013FC, 0x000013F4}, -{0x000013FD, 0x000013F5}, {0x00001C80, 0x00000412}, {0x00001C81, 0x00000414}, {0x00001C82, 0x0000041E}, -{0x00001C83, 0x00000421}, {0x00001C84, 0x00000422}, {0x00001C85, 0x00000422}, {0x00001C86, 0x0000042A}, -{0x00001C87, 0x00000462}, {0x00001C88, 0x0000A64A}, {0x00001D79, 0x0000A77D}, {0x00001D7D, 0x00002C63}, -{0x00001D8E, 0x0000A7C6}, {0x00001E01, 0x00001E00}, {0x00001E03, 0x00001E02}, {0x00001E05, 0x00001E04}, -{0x00001E07, 0x00001E06}, {0x00001E09, 0x00001E08}, {0x00001E0B, 0x00001E0A}, {0x00001E0D, 0x00001E0C}, -{0x00001E0F, 0x00001E0E}, {0x00001E11, 0x00001E10}, {0x00001E13, 0x00001E12}, {0x00001E15, 0x00001E14}, -{0x00001E17, 0x00001E16}, {0x00001E19, 0x00001E18}, {0x00001E1B, 0x00001E1A}, {0x00001E1D, 0x00001E1C}, -{0x00001E1F, 0x00001E1E}, {0x00001E21, 0x00001E20}, {0x00001E23, 0x00001E22}, {0x00001E25, 0x00001E24}, -{0x00001E27, 0x00001E26}, {0x00001E29, 0x00001E28}, {0x00001E2B, 0x00001E2A}, {0x00001E2D, 0x00001E2C}, -{0x00001E2F, 0x00001E2E}, {0x00001E31, 0x00001E30}, {0x00001E33, 0x00001E32}, {0x00001E35, 0x00001E34}, -{0x00001E37, 0x00001E36}, {0x00001E39, 0x00001E38}, {0x00001E3B, 0x00001E3A}, {0x00001E3D, 0x00001E3C}, -{0x00001E3F, 0x00001E3E}, {0x00001E41, 0x00001E40}, {0x00001E43, 0x00001E42}, {0x00001E45, 0x00001E44}, -{0x00001E47, 0x00001E46}, {0x00001E49, 0x00001E48}, {0x00001E4B, 0x00001E4A}, {0x00001E4D, 0x00001E4C}, -{0x00001E4F, 0x00001E4E}, {0x00001E51, 0x00001E50}, {0x00001E53, 0x00001E52}, {0x00001E55, 0x00001E54}, -{0x00001E57, 0x00001E56}, {0x00001E59, 0x00001E58}, {0x00001E5B, 0x00001E5A}, {0x00001E5D, 0x00001E5C}, -{0x00001E5F, 0x00001E5E}, {0x00001E61, 0x00001E60}, {0x00001E63, 0x00001E62}, {0x00001E65, 0x00001E64}, -{0x00001E67, 0x00001E66}, {0x00001E69, 0x00001E68}, {0x00001E6B, 0x00001E6A}, {0x00001E6D, 0x00001E6C}, -{0x00001E6F, 0x00001E6E}, {0x00001E71, 0x00001E70}, {0x00001E73, 0x00001E72}, {0x00001E75, 0x00001E74}, -{0x00001E77, 0x00001E76}, {0x00001E79, 0x00001E78}, {0x00001E7B, 0x00001E7A}, {0x00001E7D, 0x00001E7C}, -{0x00001E7F, 0x00001E7E}, {0x00001E81, 0x00001E80}, {0x00001E83, 0x00001E82}, {0x00001E85, 0x00001E84}, -{0x00001E87, 0x00001E86}, {0x00001E89, 0x00001E88}, {0x00001E8B, 0x00001E8A}, {0x00001E8D, 0x00001E8C}, -{0x00001E8F, 0x00001E8E}, {0x00001E91, 0x00001E90}, {0x00001E93, 0x00001E92}, {0x00001E95, 0x00001E94}, -{0x00001E96, 0x00000048}, {0x00001E97, 0x00000054}, {0x00001E98, 0x00000057}, {0x00001E99, 0x00000059}, -{0x00001E9A, 0x00000041}, {0x00001E9B, 0x00001E60}, {0x00001EA1, 0x00001EA0}, {0x00001EA3, 0x00001EA2}, -{0x00001EA5, 0x00001EA4}, {0x00001EA7, 0x00001EA6}, {0x00001EA9, 0x00001EA8}, {0x00001EAB, 0x00001EAA}, -{0x00001EAD, 0x00001EAC}, {0x00001EAF, 0x00001EAE}, {0x00001EB1, 0x00001EB0}, {0x00001EB3, 0x00001EB2}, -{0x00001EB5, 0x00001EB4}, {0x00001EB7, 0x00001EB6}, {0x00001EB9, 0x00001EB8}, {0x00001EBB, 0x00001EBA}, -{0x00001EBD, 0x00001EBC}, {0x00001EBF, 0x00001EBE}, {0x00001EC1, 0x00001EC0}, {0x00001EC3, 0x00001EC2}, -{0x00001EC5, 0x00001EC4}, {0x00001EC7, 0x00001EC6}, {0x00001EC9, 0x00001EC8}, {0x00001ECB, 0x00001ECA}, -{0x00001ECD, 0x00001ECC}, {0x00001ECF, 0x00001ECE}, {0x00001ED1, 0x00001ED0}, {0x00001ED3, 0x00001ED2}, -{0x00001ED5, 0x00001ED4}, {0x00001ED7, 0x00001ED6}, {0x00001ED9, 0x00001ED8}, {0x00001EDB, 0x00001EDA}, -{0x00001EDD, 0x00001EDC}, {0x00001EDF, 0x00001EDE}, {0x00001EE1, 0x00001EE0}, {0x00001EE3, 0x00001EE2}, -{0x00001EE5, 0x00001EE4}, {0x00001EE7, 0x00001EE6}, {0x00001EE9, 0x00001EE8}, {0x00001EEB, 0x00001EEA}, -{0x00001EED, 0x00001EEC}, {0x00001EEF, 0x00001EEE}, {0x00001EF1, 0x00001EF0}, {0x00001EF3, 0x00001EF2}, -{0x00001EF5, 0x00001EF4}, {0x00001EF7, 0x00001EF6}, {0x00001EF9, 0x00001EF8}, {0x00001EFB, 0x00001EFA}, -{0x00001EFD, 0x00001EFC}, {0x00001EFF, 0x00001EFE}, {0x00001F00, 0x00001F08}, {0x00001F01, 0x00001F09}, -{0x00001F02, 0x00001F0A}, {0x00001F03, 0x00001F0B}, {0x00001F04, 0x00001F0C}, {0x00001F05, 0x00001F0D}, -{0x00001F06, 0x00001F0E}, {0x00001F07, 0x00001F0F}, {0x00001F10, 0x00001F18}, {0x00001F11, 0x00001F19}, -{0x00001F12, 0x00001F1A}, {0x00001F13, 0x00001F1B}, {0x00001F14, 0x00001F1C}, {0x00001F15, 0x00001F1D}, -{0x00001F20, 0x00001F28}, {0x00001F21, 0x00001F29}, {0x00001F22, 0x00001F2A}, {0x00001F23, 0x00001F2B}, -{0x00001F24, 0x00001F2C}, {0x00001F25, 0x00001F2D}, {0x00001F26, 0x00001F2E}, {0x00001F27, 0x00001F2F}, -{0x00001F30, 0x00001F38}, {0x00001F31, 0x00001F39}, {0x00001F32, 0x00001F3A}, {0x00001F33, 0x00001F3B}, -{0x00001F34, 0x00001F3C}, {0x00001F35, 0x00001F3D}, {0x00001F36, 0x00001F3E}, {0x00001F37, 0x00001F3F}, -{0x00001F40, 0x00001F48}, {0x00001F41, 0x00001F49}, {0x00001F42, 0x00001F4A}, {0x00001F43, 0x00001F4B}, -{0x00001F44, 0x00001F4C}, {0x00001F45, 0x00001F4D}, {0x00001F50, 0x000003A5}, {0x00001F51, 0x00001F59}, -{0x00001F52, 0x000003A5}, {0x00001F53, 0x00001F5B}, {0x00001F54, 0x000003A5}, {0x00001F55, 0x00001F5D}, -{0x00001F56, 0x000003A5}, {0x00001F57, 0x00001F5F}, {0x00001F60, 0x00001F68}, {0x00001F61, 0x00001F69}, -{0x00001F62, 0x00001F6A}, {0x00001F63, 0x00001F6B}, {0x00001F64, 0x00001F6C}, {0x00001F65, 0x00001F6D}, -{0x00001F66, 0x00001F6E}, {0x00001F67, 0x00001F6F}, {0x00001F70, 0x00001FBA}, {0x00001F71, 0x00001FBB}, -{0x00001F72, 0x00001FC8}, {0x00001F73, 0x00001FC9}, {0x00001F74, 0x00001FCA}, {0x00001F75, 0x00001FCB}, -{0x00001F76, 0x00001FDA}, {0x00001F77, 0x00001FDB}, {0x00001F78, 0x00001FF8}, {0x00001F79, 0x00001FF9}, -{0x00001F7A, 0x00001FEA}, {0x00001F7B, 0x00001FEB}, {0x00001F7C, 0x00001FFA}, {0x00001F7D, 0x00001FFB}, -{0x00001F80, 0x00001F08}, {0x00001F81, 0x00001F09}, {0x00001F82, 0x00001F0A}, {0x00001F83, 0x00001F0B}, -{0x00001F84, 0x00001F0C}, {0x00001F85, 0x00001F0D}, {0x00001F86, 0x00001F0E}, {0x00001F87, 0x00001F0F}, -{0x00001F88, 0x00001F08}, {0x00001F89, 0x00001F09}, {0x00001F8A, 0x00001F0A}, {0x00001F8B, 0x00001F0B}, -{0x00001F8C, 0x00001F0C}, {0x00001F8D, 0x00001F0D}, {0x00001F8E, 0x00001F0E}, {0x00001F8F, 0x00001F0F}, -{0x00001F90, 0x00001F28}, {0x00001F91, 0x00001F29}, {0x00001F92, 0x00001F2A}, {0x00001F93, 0x00001F2B}, -{0x00001F94, 0x00001F2C}, {0x00001F95, 0x00001F2D}, {0x00001F96, 0x00001F2E}, {0x00001F97, 0x00001F2F}, -{0x00001F98, 0x00001F28}, {0x00001F99, 0x00001F29}, {0x00001F9A, 0x00001F2A}, {0x00001F9B, 0x00001F2B}, -{0x00001F9C, 0x00001F2C}, {0x00001F9D, 0x00001F2D}, {0x00001F9E, 0x00001F2E}, {0x00001F9F, 0x00001F2F}, -{0x00001FA0, 0x00001F68}, {0x00001FA1, 0x00001F69}, {0x00001FA2, 0x00001F6A}, {0x00001FA3, 0x00001F6B}, -{0x00001FA4, 0x00001F6C}, {0x00001FA5, 0x00001F6D}, {0x00001FA6, 0x00001F6E}, {0x00001FA7, 0x00001F6F}, -{0x00001FA8, 0x00001F68}, {0x00001FA9, 0x00001F69}, {0x00001FAA, 0x00001F6A}, {0x00001FAB, 0x00001F6B}, -{0x00001FAC, 0x00001F6C}, {0x00001FAD, 0x00001F6D}, {0x00001FAE, 0x00001F6E}, {0x00001FAF, 0x00001F6F}, -{0x00001FB0, 0x00001FB8}, {0x00001FB1, 0x00001FB9}, {0x00001FB2, 0x00001FBA}, {0x00001FB3, 0x00000391}, -{0x00001FB4, 0x00000386}, {0x00001FB6, 0x00000391}, {0x00001FB7, 0x00000391}, {0x00001FBC, 0x00000391}, -{0x00001FBE, 0x00000399}, {0x00001FC2, 0x00001FCA}, {0x00001FC3, 0x00000397}, {0x00001FC4, 0x00000389}, -{0x00001FC6, 0x00000397}, {0x00001FC7, 0x00000397}, {0x00001FCC, 0x00000397}, {0x00001FD0, 0x00001FD8}, -{0x00001FD1, 0x00001FD9}, {0x00001FD2, 0x00000399}, {0x00001FD3, 0x00000399}, {0x00001FD6, 0x00000399}, -{0x00001FD7, 0x00000399}, {0x00001FE0, 0x00001FE8}, {0x00001FE1, 0x00001FE9}, {0x00001FE2, 0x000003A5}, -{0x00001FE3, 0x000003A5}, {0x00001FE4, 0x000003A1}, {0x00001FE5, 0x00001FEC}, {0x00001FE6, 0x000003A5}, -{0x00001FE7, 0x000003A5}, {0x00001FF2, 0x00001FFA}, {0x00001FF3, 0x000003A9}, {0x00001FF4, 0x0000038F}, -{0x00001FF6, 0x000003A9}, {0x00001FF7, 0x000003A9}, {0x00001FFC, 0x000003A9}, {0x0000214E, 0x00002132}, -{0x00002170, 0x00002160}, {0x00002171, 0x00002161}, {0x00002172, 0x00002162}, {0x00002173, 0x00002163}, -{0x00002174, 0x00002164}, {0x00002175, 0x00002165}, {0x00002176, 0x00002166}, {0x00002177, 0x00002167}, -{0x00002178, 0x00002168}, {0x00002179, 0x00002169}, {0x0000217A, 0x0000216A}, {0x0000217B, 0x0000216B}, -{0x0000217C, 0x0000216C}, {0x0000217D, 0x0000216D}, {0x0000217E, 0x0000216E}, {0x0000217F, 0x0000216F}, -{0x00002184, 0x00002183}, {0x000024D0, 0x000024B6}, {0x000024D1, 0x000024B7}, {0x000024D2, 0x000024B8}, -{0x000024D3, 0x000024B9}, {0x000024D4, 0x000024BA}, {0x000024D5, 0x000024BB}, {0x000024D6, 0x000024BC}, -{0x000024D7, 0x000024BD}, {0x000024D8, 0x000024BE}, {0x000024D9, 0x000024BF}, {0x000024DA, 0x000024C0}, -{0x000024DB, 0x000024C1}, {0x000024DC, 0x000024C2}, {0x000024DD, 0x000024C3}, {0x000024DE, 0x000024C4}, -{0x000024DF, 0x000024C5}, {0x000024E0, 0x000024C6}, {0x000024E1, 0x000024C7}, {0x000024E2, 0x000024C8}, -{0x000024E3, 0x000024C9}, {0x000024E4, 0x000024CA}, {0x000024E5, 0x000024CB}, {0x000024E6, 0x000024CC}, -{0x000024E7, 0x000024CD}, {0x000024E8, 0x000024CE}, {0x000024E9, 0x000024CF}, {0x00002C30, 0x00002C00}, -{0x00002C31, 0x00002C01}, {0x00002C32, 0x00002C02}, {0x00002C33, 0x00002C03}, {0x00002C34, 0x00002C04}, -{0x00002C35, 0x00002C05}, {0x00002C36, 0x00002C06}, {0x00002C37, 0x00002C07}, {0x00002C38, 0x00002C08}, -{0x00002C39, 0x00002C09}, {0x00002C3A, 0x00002C0A}, {0x00002C3B, 0x00002C0B}, {0x00002C3C, 0x00002C0C}, -{0x00002C3D, 0x00002C0D}, {0x00002C3E, 0x00002C0E}, {0x00002C3F, 0x00002C0F}, {0x00002C40, 0x00002C10}, -{0x00002C41, 0x00002C11}, {0x00002C42, 0x00002C12}, {0x00002C43, 0x00002C13}, {0x00002C44, 0x00002C14}, -{0x00002C45, 0x00002C15}, {0x00002C46, 0x00002C16}, {0x00002C47, 0x00002C17}, {0x00002C48, 0x00002C18}, -{0x00002C49, 0x00002C19}, {0x00002C4A, 0x00002C1A}, {0x00002C4B, 0x00002C1B}, {0x00002C4C, 0x00002C1C}, -{0x00002C4D, 0x00002C1D}, {0x00002C4E, 0x00002C1E}, {0x00002C4F, 0x00002C1F}, {0x00002C50, 0x00002C20}, -{0x00002C51, 0x00002C21}, {0x00002C52, 0x00002C22}, {0x00002C53, 0x00002C23}, {0x00002C54, 0x00002C24}, -{0x00002C55, 0x00002C25}, {0x00002C56, 0x00002C26}, {0x00002C57, 0x00002C27}, {0x00002C58, 0x00002C28}, -{0x00002C59, 0x00002C29}, {0x00002C5A, 0x00002C2A}, {0x00002C5B, 0x00002C2B}, {0x00002C5C, 0x00002C2C}, -{0x00002C5D, 0x00002C2D}, {0x00002C5E, 0x00002C2E}, {0x00002C61, 0x00002C60}, {0x00002C65, 0x0000023A}, -{0x00002C66, 0x0000023E}, {0x00002C68, 0x00002C67}, {0x00002C6A, 0x00002C69}, {0x00002C6C, 0x00002C6B}, -{0x00002C73, 0x00002C72}, {0x00002C76, 0x00002C75}, {0x00002C81, 0x00002C80}, {0x00002C83, 0x00002C82}, -{0x00002C85, 0x00002C84}, {0x00002C87, 0x00002C86}, {0x00002C89, 0x00002C88}, {0x00002C8B, 0x00002C8A}, -{0x00002C8D, 0x00002C8C}, {0x00002C8F, 0x00002C8E}, {0x00002C91, 0x00002C90}, {0x00002C93, 0x00002C92}, -{0x00002C95, 0x00002C94}, {0x00002C97, 0x00002C96}, {0x00002C99, 0x00002C98}, {0x00002C9B, 0x00002C9A}, -{0x00002C9D, 0x00002C9C}, {0x00002C9F, 0x00002C9E}, {0x00002CA1, 0x00002CA0}, {0x00002CA3, 0x00002CA2}, -{0x00002CA5, 0x00002CA4}, {0x00002CA7, 0x00002CA6}, {0x00002CA9, 0x00002CA8}, {0x00002CAB, 0x00002CAA}, -{0x00002CAD, 0x00002CAC}, {0x00002CAF, 0x00002CAE}, {0x00002CB1, 0x00002CB0}, {0x00002CB3, 0x00002CB2}, -{0x00002CB5, 0x00002CB4}, {0x00002CB7, 0x00002CB6}, {0x00002CB9, 0x00002CB8}, {0x00002CBB, 0x00002CBA}, -{0x00002CBD, 0x00002CBC}, {0x00002CBF, 0x00002CBE}, {0x00002CC1, 0x00002CC0}, {0x00002CC3, 0x00002CC2}, -{0x00002CC5, 0x00002CC4}, {0x00002CC7, 0x00002CC6}, {0x00002CC9, 0x00002CC8}, {0x00002CCB, 0x00002CCA}, -{0x00002CCD, 0x00002CCC}, {0x00002CCF, 0x00002CCE}, {0x00002CD1, 0x00002CD0}, {0x00002CD3, 0x00002CD2}, -{0x00002CD5, 0x00002CD4}, {0x00002CD7, 0x00002CD6}, {0x00002CD9, 0x00002CD8}, {0x00002CDB, 0x00002CDA}, -{0x00002CDD, 0x00002CDC}, {0x00002CDF, 0x00002CDE}, {0x00002CE1, 0x00002CE0}, {0x00002CE3, 0x00002CE2}, -{0x00002CEC, 0x00002CEB}, {0x00002CEE, 0x00002CED}, {0x00002CF3, 0x00002CF2}, {0x00002D00, 0x000010A0}, -{0x00002D01, 0x000010A1}, {0x00002D02, 0x000010A2}, {0x00002D03, 0x000010A3}, {0x00002D04, 0x000010A4}, -{0x00002D05, 0x000010A5}, {0x00002D06, 0x000010A6}, {0x00002D07, 0x000010A7}, {0x00002D08, 0x000010A8}, -{0x00002D09, 0x000010A9}, {0x00002D0A, 0x000010AA}, {0x00002D0B, 0x000010AB}, {0x00002D0C, 0x000010AC}, -{0x00002D0D, 0x000010AD}, {0x00002D0E, 0x000010AE}, {0x00002D0F, 0x000010AF}, {0x00002D10, 0x000010B0}, -{0x00002D11, 0x000010B1}, {0x00002D12, 0x000010B2}, {0x00002D13, 0x000010B3}, {0x00002D14, 0x000010B4}, -{0x00002D15, 0x000010B5}, {0x00002D16, 0x000010B6}, {0x00002D17, 0x000010B7}, {0x00002D18, 0x000010B8}, -{0x00002D19, 0x000010B9}, {0x00002D1A, 0x000010BA}, {0x00002D1B, 0x000010BB}, {0x00002D1C, 0x000010BC}, -{0x00002D1D, 0x000010BD}, {0x00002D1E, 0x000010BE}, {0x00002D1F, 0x000010BF}, {0x00002D20, 0x000010C0}, -{0x00002D21, 0x000010C1}, {0x00002D22, 0x000010C2}, {0x00002D23, 0x000010C3}, {0x00002D24, 0x000010C4}, -{0x00002D25, 0x000010C5}, {0x00002D27, 0x000010C7}, {0x00002D2D, 0x000010CD}, {0x0000A641, 0x0000A640}, -{0x0000A643, 0x0000A642}, {0x0000A645, 0x0000A644}, {0x0000A647, 0x0000A646}, {0x0000A649, 0x0000A648}, -{0x0000A64B, 0x0000A64A}, {0x0000A64D, 0x0000A64C}, {0x0000A64F, 0x0000A64E}, {0x0000A651, 0x0000A650}, -{0x0000A653, 0x0000A652}, {0x0000A655, 0x0000A654}, {0x0000A657, 0x0000A656}, {0x0000A659, 0x0000A658}, -{0x0000A65B, 0x0000A65A}, {0x0000A65D, 0x0000A65C}, {0x0000A65F, 0x0000A65E}, {0x0000A661, 0x0000A660}, -{0x0000A663, 0x0000A662}, {0x0000A665, 0x0000A664}, {0x0000A667, 0x0000A666}, {0x0000A669, 0x0000A668}, -{0x0000A66B, 0x0000A66A}, {0x0000A66D, 0x0000A66C}, {0x0000A681, 0x0000A680}, {0x0000A683, 0x0000A682}, -{0x0000A685, 0x0000A684}, {0x0000A687, 0x0000A686}, {0x0000A689, 0x0000A688}, {0x0000A68B, 0x0000A68A}, -{0x0000A68D, 0x0000A68C}, {0x0000A68F, 0x0000A68E}, {0x0000A691, 0x0000A690}, {0x0000A693, 0x0000A692}, -{0x0000A695, 0x0000A694}, {0x0000A697, 0x0000A696}, {0x0000A699, 0x0000A698}, {0x0000A69B, 0x0000A69A}, -{0x0000A723, 0x0000A722}, {0x0000A725, 0x0000A724}, {0x0000A727, 0x0000A726}, {0x0000A729, 0x0000A728}, -{0x0000A72B, 0x0000A72A}, {0x0000A72D, 0x0000A72C}, {0x0000A72F, 0x0000A72E}, {0x0000A733, 0x0000A732}, -{0x0000A735, 0x0000A734}, {0x0000A737, 0x0000A736}, {0x0000A739, 0x0000A738}, {0x0000A73B, 0x0000A73A}, -{0x0000A73D, 0x0000A73C}, {0x0000A73F, 0x0000A73E}, {0x0000A741, 0x0000A740}, {0x0000A743, 0x0000A742}, -{0x0000A745, 0x0000A744}, {0x0000A747, 0x0000A746}, {0x0000A749, 0x0000A748}, {0x0000A74B, 0x0000A74A}, -{0x0000A74D, 0x0000A74C}, {0x0000A74F, 0x0000A74E}, {0x0000A751, 0x0000A750}, {0x0000A753, 0x0000A752}, -{0x0000A755, 0x0000A754}, {0x0000A757, 0x0000A756}, {0x0000A759, 0x0000A758}, {0x0000A75B, 0x0000A75A}, -{0x0000A75D, 0x0000A75C}, {0x0000A75F, 0x0000A75E}, {0x0000A761, 0x0000A760}, {0x0000A763, 0x0000A762}, -{0x0000A765, 0x0000A764}, {0x0000A767, 0x0000A766}, {0x0000A769, 0x0000A768}, {0x0000A76B, 0x0000A76A}, -{0x0000A76D, 0x0000A76C}, {0x0000A76F, 0x0000A76E}, {0x0000A77A, 0x0000A779}, {0x0000A77C, 0x0000A77B}, -{0x0000A77F, 0x0000A77E}, {0x0000A781, 0x0000A780}, {0x0000A783, 0x0000A782}, {0x0000A785, 0x0000A784}, -{0x0000A787, 0x0000A786}, {0x0000A78C, 0x0000A78B}, {0x0000A791, 0x0000A790}, {0x0000A793, 0x0000A792}, -{0x0000A794, 0x0000A7C4}, {0x0000A797, 0x0000A796}, {0x0000A799, 0x0000A798}, {0x0000A79B, 0x0000A79A}, -{0x0000A79D, 0x0000A79C}, {0x0000A79F, 0x0000A79E}, {0x0000A7A1, 0x0000A7A0}, {0x0000A7A3, 0x0000A7A2}, -{0x0000A7A5, 0x0000A7A4}, {0x0000A7A7, 0x0000A7A6}, {0x0000A7A9, 0x0000A7A8}, {0x0000A7B5, 0x0000A7B4}, -{0x0000A7B7, 0x0000A7B6}, {0x0000A7B9, 0x0000A7B8}, {0x0000A7BB, 0x0000A7BA}, {0x0000A7BD, 0x0000A7BC}, -{0x0000A7BF, 0x0000A7BE}, {0x0000A7C3, 0x0000A7C2}, {0x0000A7C8, 0x0000A7C7}, {0x0000A7CA, 0x0000A7C9}, -{0x0000A7F6, 0x0000A7F5}, {0x0000AB53, 0x0000A7B3}, {0x0000AB70, 0x000013A0}, {0x0000AB71, 0x000013A1}, -{0x0000AB72, 0x000013A2}, {0x0000AB73, 0x000013A3}, {0x0000AB74, 0x000013A4}, {0x0000AB75, 0x000013A5}, -{0x0000AB76, 0x000013A6}, {0x0000AB77, 0x000013A7}, {0x0000AB78, 0x000013A8}, {0x0000AB79, 0x000013A9}, -{0x0000AB7A, 0x000013AA}, {0x0000AB7B, 0x000013AB}, {0x0000AB7C, 0x000013AC}, {0x0000AB7D, 0x000013AD}, -{0x0000AB7E, 0x000013AE}, {0x0000AB7F, 0x000013AF}, {0x0000AB80, 0x000013B0}, {0x0000AB81, 0x000013B1}, -{0x0000AB82, 0x000013B2}, {0x0000AB83, 0x000013B3}, {0x0000AB84, 0x000013B4}, {0x0000AB85, 0x000013B5}, -{0x0000AB86, 0x000013B6}, {0x0000AB87, 0x000013B7}, {0x0000AB88, 0x000013B8}, {0x0000AB89, 0x000013B9}, -{0x0000AB8A, 0x000013BA}, {0x0000AB8B, 0x000013BB}, {0x0000AB8C, 0x000013BC}, {0x0000AB8D, 0x000013BD}, -{0x0000AB8E, 0x000013BE}, {0x0000AB8F, 0x000013BF}, {0x0000AB90, 0x000013C0}, {0x0000AB91, 0x000013C1}, -{0x0000AB92, 0x000013C2}, {0x0000AB93, 0x000013C3}, {0x0000AB94, 0x000013C4}, {0x0000AB95, 0x000013C5}, -{0x0000AB96, 0x000013C6}, {0x0000AB97, 0x000013C7}, {0x0000AB98, 0x000013C8}, {0x0000AB99, 0x000013C9}, -{0x0000AB9A, 0x000013CA}, {0x0000AB9B, 0x000013CB}, {0x0000AB9C, 0x000013CC}, {0x0000AB9D, 0x000013CD}, -{0x0000AB9E, 0x000013CE}, {0x0000AB9F, 0x000013CF}, {0x0000ABA0, 0x000013D0}, {0x0000ABA1, 0x000013D1}, -{0x0000ABA2, 0x000013D2}, {0x0000ABA3, 0x000013D3}, {0x0000ABA4, 0x000013D4}, {0x0000ABA5, 0x000013D5}, -{0x0000ABA6, 0x000013D6}, {0x0000ABA7, 0x000013D7}, {0x0000ABA8, 0x000013D8}, {0x0000ABA9, 0x000013D9}, -{0x0000ABAA, 0x000013DA}, {0x0000ABAB, 0x000013DB}, {0x0000ABAC, 0x000013DC}, {0x0000ABAD, 0x000013DD}, -{0x0000ABAE, 0x000013DE}, {0x0000ABAF, 0x000013DF}, {0x0000ABB0, 0x000013E0}, {0x0000ABB1, 0x000013E1}, -{0x0000ABB2, 0x000013E2}, {0x0000ABB3, 0x000013E3}, {0x0000ABB4, 0x000013E4}, {0x0000ABB5, 0x000013E5}, -{0x0000ABB6, 0x000013E6}, {0x0000ABB7, 0x000013E7}, {0x0000ABB8, 0x000013E8}, {0x0000ABB9, 0x000013E9}, -{0x0000ABBA, 0x000013EA}, {0x0000ABBB, 0x000013EB}, {0x0000ABBC, 0x000013EC}, {0x0000ABBD, 0x000013ED}, -{0x0000ABBE, 0x000013EE}, {0x0000ABBF, 0x000013EF}, {0x0000FB00, 0x00000046}, {0x0000FB01, 0x00000046}, -{0x0000FB02, 0x00000046}, {0x0000FB03, 0x00000046}, {0x0000FB04, 0x00000046}, {0x0000FB05, 0x00000053}, -{0x0000FB06, 0x00000053}, {0x0000FB13, 0x00000544}, {0x0000FB14, 0x00000544}, {0x0000FB15, 0x00000544}, -{0x0000FB16, 0x0000054E}, {0x0000FB17, 0x00000544}, {0x0000FF41, 0x0000FF21}, {0x0000FF42, 0x0000FF22}, -{0x0000FF43, 0x0000FF23}, {0x0000FF44, 0x0000FF24}, {0x0000FF45, 0x0000FF25}, {0x0000FF46, 0x0000FF26}, -{0x0000FF47, 0x0000FF27}, {0x0000FF48, 0x0000FF28}, {0x0000FF49, 0x0000FF29}, {0x0000FF4A, 0x0000FF2A}, -{0x0000FF4B, 0x0000FF2B}, {0x0000FF4C, 0x0000FF2C}, {0x0000FF4D, 0x0000FF2D}, {0x0000FF4E, 0x0000FF2E}, -{0x0000FF4F, 0x0000FF2F}, {0x0000FF50, 0x0000FF30}, {0x0000FF51, 0x0000FF31}, {0x0000FF52, 0x0000FF32}, -{0x0000FF53, 0x0000FF33}, {0x0000FF54, 0x0000FF34}, {0x0000FF55, 0x0000FF35}, {0x0000FF56, 0x0000FF36}, -{0x0000FF57, 0x0000FF37}, {0x0000FF58, 0x0000FF38}, {0x0000FF59, 0x0000FF39}, {0x0000FF5A, 0x0000FF3A}, -{0x00010428, 0x00010400}, {0x00010429, 0x00010401}, {0x0001042A, 0x00010402}, {0x0001042B, 0x00010403}, -{0x0001042C, 0x00010404}, {0x0001042D, 0x00010405}, {0x0001042E, 0x00010406}, {0x0001042F, 0x00010407}, -{0x00010430, 0x00010408}, {0x00010431, 0x00010409}, {0x00010432, 0x0001040A}, {0x00010433, 0x0001040B}, -{0x00010434, 0x0001040C}, {0x00010435, 0x0001040D}, {0x00010436, 0x0001040E}, {0x00010437, 0x0001040F}, -{0x00010438, 0x00010410}, {0x00010439, 0x00010411}, {0x0001043A, 0x00010412}, {0x0001043B, 0x00010413}, -{0x0001043C, 0x00010414}, {0x0001043D, 0x00010415}, {0x0001043E, 0x00010416}, {0x0001043F, 0x00010417}, -{0x00010440, 0x00010418}, {0x00010441, 0x00010419}, {0x00010442, 0x0001041A}, {0x00010443, 0x0001041B}, -{0x00010444, 0x0001041C}, {0x00010445, 0x0001041D}, {0x00010446, 0x0001041E}, {0x00010447, 0x0001041F}, -{0x00010448, 0x00010420}, {0x00010449, 0x00010421}, {0x0001044A, 0x00010422}, {0x0001044B, 0x00010423}, -{0x0001044C, 0x00010424}, {0x0001044D, 0x00010425}, {0x0001044E, 0x00010426}, {0x0001044F, 0x00010427}, -{0x000104D8, 0x000104B0}, {0x000104D9, 0x000104B1}, {0x000104DA, 0x000104B2}, {0x000104DB, 0x000104B3}, -{0x000104DC, 0x000104B4}, {0x000104DD, 0x000104B5}, {0x000104DE, 0x000104B6}, {0x000104DF, 0x000104B7}, -{0x000104E0, 0x000104B8}, {0x000104E1, 0x000104B9}, {0x000104E2, 0x000104BA}, {0x000104E3, 0x000104BB}, -{0x000104E4, 0x000104BC}, {0x000104E5, 0x000104BD}, {0x000104E6, 0x000104BE}, {0x000104E7, 0x000104BF}, -{0x000104E8, 0x000104C0}, {0x000104E9, 0x000104C1}, {0x000104EA, 0x000104C2}, {0x000104EB, 0x000104C3}, -{0x000104EC, 0x000104C4}, {0x000104ED, 0x000104C5}, {0x000104EE, 0x000104C6}, {0x000104EF, 0x000104C7}, -{0x000104F0, 0x000104C8}, {0x000104F1, 0x000104C9}, {0x000104F2, 0x000104CA}, {0x000104F3, 0x000104CB}, -{0x000104F4, 0x000104CC}, {0x000104F5, 0x000104CD}, {0x000104F6, 0x000104CE}, {0x000104F7, 0x000104CF}, -{0x000104F8, 0x000104D0}, {0x000104F9, 0x000104D1}, {0x000104FA, 0x000104D2}, {0x000104FB, 0x000104D3}, -{0x00010CC0, 0x00010C80}, {0x00010CC1, 0x00010C81}, {0x00010CC2, 0x00010C82}, {0x00010CC3, 0x00010C83}, -{0x00010CC4, 0x00010C84}, {0x00010CC5, 0x00010C85}, {0x00010CC6, 0x00010C86}, {0x00010CC7, 0x00010C87}, -{0x00010CC8, 0x00010C88}, {0x00010CC9, 0x00010C89}, {0x00010CCA, 0x00010C8A}, {0x00010CCB, 0x00010C8B}, -{0x00010CCC, 0x00010C8C}, {0x00010CCD, 0x00010C8D}, {0x00010CCE, 0x00010C8E}, {0x00010CCF, 0x00010C8F}, -{0x00010CD0, 0x00010C90}, {0x00010CD1, 0x00010C91}, {0x00010CD2, 0x00010C92}, {0x00010CD3, 0x00010C93}, -{0x00010CD4, 0x00010C94}, {0x00010CD5, 0x00010C95}, {0x00010CD6, 0x00010C96}, {0x00010CD7, 0x00010C97}, -{0x00010CD8, 0x00010C98}, {0x00010CD9, 0x00010C99}, {0x00010CDA, 0x00010C9A}, {0x00010CDB, 0x00010C9B}, -{0x00010CDC, 0x00010C9C}, {0x00010CDD, 0x00010C9D}, {0x00010CDE, 0x00010C9E}, {0x00010CDF, 0x00010C9F}, -{0x00010CE0, 0x00010CA0}, {0x00010CE1, 0x00010CA1}, {0x00010CE2, 0x00010CA2}, {0x00010CE3, 0x00010CA3}, -{0x00010CE4, 0x00010CA4}, {0x00010CE5, 0x00010CA5}, {0x00010CE6, 0x00010CA6}, {0x00010CE7, 0x00010CA7}, -{0x00010CE8, 0x00010CA8}, {0x00010CE9, 0x00010CA9}, {0x00010CEA, 0x00010CAA}, {0x00010CEB, 0x00010CAB}, -{0x00010CEC, 0x00010CAC}, {0x00010CED, 0x00010CAD}, {0x00010CEE, 0x00010CAE}, {0x00010CEF, 0x00010CAF}, -{0x00010CF0, 0x00010CB0}, {0x00010CF1, 0x00010CB1}, {0x00010CF2, 0x00010CB2}, {0x000118C0, 0x000118A0}, -{0x000118C1, 0x000118A1}, {0x000118C2, 0x000118A2}, {0x000118C3, 0x000118A3}, {0x000118C4, 0x000118A4}, -{0x000118C5, 0x000118A5}, {0x000118C6, 0x000118A6}, {0x000118C7, 0x000118A7}, {0x000118C8, 0x000118A8}, -{0x000118C9, 0x000118A9}, {0x000118CA, 0x000118AA}, {0x000118CB, 0x000118AB}, {0x000118CC, 0x000118AC}, -{0x000118CD, 0x000118AD}, {0x000118CE, 0x000118AE}, {0x000118CF, 0x000118AF}, {0x000118D0, 0x000118B0}, -{0x000118D1, 0x000118B1}, {0x000118D2, 0x000118B2}, {0x000118D3, 0x000118B3}, {0x000118D4, 0x000118B4}, -{0x000118D5, 0x000118B5}, {0x000118D6, 0x000118B6}, {0x000118D7, 0x000118B7}, {0x000118D8, 0x000118B8}, -{0x000118D9, 0x000118B9}, {0x000118DA, 0x000118BA}, {0x000118DB, 0x000118BB}, {0x000118DC, 0x000118BC}, -{0x000118DD, 0x000118BD}, {0x000118DE, 0x000118BE}, {0x000118DF, 0x000118BF}, {0x00016E60, 0x00016E40}, -{0x00016E61, 0x00016E41}, {0x00016E62, 0x00016E42}, {0x00016E63, 0x00016E43}, {0x00016E64, 0x00016E44}, -{0x00016E65, 0x00016E45}, {0x00016E66, 0x00016E46}, {0x00016E67, 0x00016E47}, {0x00016E68, 0x00016E48}, -{0x00016E69, 0x00016E49}, {0x00016E6A, 0x00016E4A}, {0x00016E6B, 0x00016E4B}, {0x00016E6C, 0x00016E4C}, -{0x00016E6D, 0x00016E4D}, {0x00016E6E, 0x00016E4E}, {0x00016E6F, 0x00016E4F}, {0x00016E70, 0x00016E50}, -{0x00016E71, 0x00016E51}, {0x00016E72, 0x00016E52}, {0x00016E73, 0x00016E53}, {0x00016E74, 0x00016E54}, -{0x00016E75, 0x00016E55}, {0x00016E76, 0x00016E56}, {0x00016E77, 0x00016E57}, {0x00016E78, 0x00016E58}, -{0x00016E79, 0x00016E59}, {0x00016E7A, 0x00016E5A}, {0x00016E7B, 0x00016E5B}, {0x00016E7C, 0x00016E5C}, -{0x00016E7D, 0x00016E5D}, {0x00016E7E, 0x00016E5E}, {0x00016E7F, 0x00016E5F}, {0x0001E922, 0x0001E900}, -{0x0001E923, 0x0001E901}, {0x0001E924, 0x0001E902}, {0x0001E925, 0x0001E903}, {0x0001E926, 0x0001E904}, -{0x0001E927, 0x0001E905}, {0x0001E928, 0x0001E906}, {0x0001E929, 0x0001E907}, {0x0001E92A, 0x0001E908}, -{0x0001E92B, 0x0001E909}, {0x0001E92C, 0x0001E90A}, {0x0001E92D, 0x0001E90B}, {0x0001E92E, 0x0001E90C}, -{0x0001E92F, 0x0001E90D}, {0x0001E930, 0x0001E90E}, {0x0001E931, 0x0001E90F}, {0x0001E932, 0x0001E910}, -{0x0001E933, 0x0001E911}, {0x0001E934, 0x0001E912}, {0x0001E935, 0x0001E913}, {0x0001E936, 0x0001E914}, -{0x0001E937, 0x0001E915}, {0x0001E938, 0x0001E916}, {0x0001E939, 0x0001E917}, {0x0001E93A, 0x0001E918}, -{0x0001E93B, 0x0001E919}, {0x0001E93C, 0x0001E91A}, {0x0001E93D, 0x0001E91B}, {0x0001E93E, 0x0001E91C}, -{0x0001E93F, 0x0001E91D}, {0x0001E940, 0x0001E91E}, {0x0001E941, 0x0001E91F}, {0x0001E942, 0x0001E920}, -{0x0001E943, 0x0001E921}, -}; - -const std::multimap unicode_map_nfd = { -{0x000000C0, 0x00000041}, {0x000000C0, 0x00000300}, {0x000000C1, 0x00000041}, {0x000000C1, 0x00000301}, -{0x000000C2, 0x00000041}, {0x000000C2, 0x00000302}, {0x000000C3, 0x00000041}, {0x000000C3, 0x00000303}, -{0x000000C4, 0x00000041}, {0x000000C4, 0x00000308}, {0x000000C5, 0x00000041}, {0x000000C5, 0x0000030A}, -{0x000000C7, 0x00000043}, {0x000000C7, 0x00000327}, {0x000000C8, 0x00000045}, {0x000000C8, 0x00000300}, -{0x000000C9, 0x00000045}, {0x000000C9, 0x00000301}, {0x000000CA, 0x00000045}, {0x000000CA, 0x00000302}, -{0x000000CB, 0x00000045}, {0x000000CB, 0x00000308}, {0x000000CC, 0x00000049}, {0x000000CC, 0x00000300}, -{0x000000CD, 0x00000049}, {0x000000CD, 0x00000301}, {0x000000CE, 0x00000049}, {0x000000CE, 0x00000302}, -{0x000000CF, 0x00000049}, {0x000000CF, 0x00000308}, {0x000000D1, 0x0000004E}, {0x000000D1, 0x00000303}, -{0x000000D2, 0x0000004F}, {0x000000D2, 0x00000300}, {0x000000D3, 0x0000004F}, {0x000000D3, 0x00000301}, -{0x000000D4, 0x0000004F}, {0x000000D4, 0x00000302}, {0x000000D5, 0x0000004F}, {0x000000D5, 0x00000303}, -{0x000000D6, 0x0000004F}, {0x000000D6, 0x00000308}, {0x000000D9, 0x00000055}, {0x000000D9, 0x00000300}, -{0x000000DA, 0x00000055}, {0x000000DA, 0x00000301}, {0x000000DB, 0x00000055}, {0x000000DB, 0x00000302}, -{0x000000DC, 0x00000055}, {0x000000DC, 0x00000308}, {0x000000DD, 0x00000059}, {0x000000DD, 0x00000301}, -{0x000000E0, 0x00000061}, {0x000000E0, 0x00000300}, {0x000000E1, 0x00000061}, {0x000000E1, 0x00000301}, -{0x000000E2, 0x00000061}, {0x000000E2, 0x00000302}, {0x000000E3, 0x00000061}, {0x000000E3, 0x00000303}, -{0x000000E4, 0x00000061}, {0x000000E4, 0x00000308}, {0x000000E5, 0x00000061}, {0x000000E5, 0x0000030A}, -{0x000000E7, 0x00000063}, {0x000000E7, 0x00000327}, {0x000000E8, 0x00000065}, {0x000000E8, 0x00000300}, -{0x000000E9, 0x00000065}, {0x000000E9, 0x00000301}, {0x000000EA, 0x00000065}, {0x000000EA, 0x00000302}, -{0x000000EB, 0x00000065}, {0x000000EB, 0x00000308}, {0x000000EC, 0x00000069}, {0x000000EC, 0x00000300}, -{0x000000ED, 0x00000069}, {0x000000ED, 0x00000301}, {0x000000EE, 0x00000069}, {0x000000EE, 0x00000302}, -{0x000000EF, 0x00000069}, {0x000000EF, 0x00000308}, {0x000000F1, 0x0000006E}, {0x000000F1, 0x00000303}, -{0x000000F2, 0x0000006F}, {0x000000F2, 0x00000300}, {0x000000F3, 0x0000006F}, {0x000000F3, 0x00000301}, -{0x000000F4, 0x0000006F}, {0x000000F4, 0x00000302}, {0x000000F5, 0x0000006F}, {0x000000F5, 0x00000303}, -{0x000000F6, 0x0000006F}, {0x000000F6, 0x00000308}, {0x000000F9, 0x00000075}, {0x000000F9, 0x00000300}, -{0x000000FA, 0x00000075}, {0x000000FA, 0x00000301}, {0x000000FB, 0x00000075}, {0x000000FB, 0x00000302}, -{0x000000FC, 0x00000075}, {0x000000FC, 0x00000308}, {0x000000FD, 0x00000079}, {0x000000FD, 0x00000301}, -{0x000000FF, 0x00000079}, {0x000000FF, 0x00000308}, {0x00000100, 0x00000041}, {0x00000100, 0x00000304}, -{0x00000101, 0x00000061}, {0x00000101, 0x00000304}, {0x00000102, 0x00000041}, {0x00000102, 0x00000306}, -{0x00000103, 0x00000061}, {0x00000103, 0x00000306}, {0x00000104, 0x00000041}, {0x00000104, 0x00000328}, -{0x00000105, 0x00000061}, {0x00000105, 0x00000328}, {0x00000106, 0x00000043}, {0x00000106, 0x00000301}, -{0x00000107, 0x00000063}, {0x00000107, 0x00000301}, {0x00000108, 0x00000043}, {0x00000108, 0x00000302}, -{0x00000109, 0x00000063}, {0x00000109, 0x00000302}, {0x0000010A, 0x00000043}, {0x0000010A, 0x00000307}, -{0x0000010B, 0x00000063}, {0x0000010B, 0x00000307}, {0x0000010C, 0x00000043}, {0x0000010C, 0x0000030C}, -{0x0000010D, 0x00000063}, {0x0000010D, 0x0000030C}, {0x0000010E, 0x00000044}, {0x0000010E, 0x0000030C}, -{0x0000010F, 0x00000064}, {0x0000010F, 0x0000030C}, {0x00000112, 0x00000045}, {0x00000112, 0x00000304}, -{0x00000113, 0x00000065}, {0x00000113, 0x00000304}, {0x00000114, 0x00000045}, {0x00000114, 0x00000306}, -{0x00000115, 0x00000065}, {0x00000115, 0x00000306}, {0x00000116, 0x00000045}, {0x00000116, 0x00000307}, -{0x00000117, 0x00000065}, {0x00000117, 0x00000307}, {0x00000118, 0x00000045}, {0x00000118, 0x00000328}, -{0x00000119, 0x00000065}, {0x00000119, 0x00000328}, {0x0000011A, 0x00000045}, {0x0000011A, 0x0000030C}, -{0x0000011B, 0x00000065}, {0x0000011B, 0x0000030C}, {0x0000011C, 0x00000047}, {0x0000011C, 0x00000302}, -{0x0000011D, 0x00000067}, {0x0000011D, 0x00000302}, {0x0000011E, 0x00000047}, {0x0000011E, 0x00000306}, -{0x0000011F, 0x00000067}, {0x0000011F, 0x00000306}, {0x00000120, 0x00000047}, {0x00000120, 0x00000307}, -{0x00000121, 0x00000067}, {0x00000121, 0x00000307}, {0x00000122, 0x00000047}, {0x00000122, 0x00000327}, -{0x00000123, 0x00000067}, {0x00000123, 0x00000327}, {0x00000124, 0x00000048}, {0x00000124, 0x00000302}, -{0x00000125, 0x00000068}, {0x00000125, 0x00000302}, {0x00000128, 0x00000049}, {0x00000128, 0x00000303}, -{0x00000129, 0x00000069}, {0x00000129, 0x00000303}, {0x0000012A, 0x00000049}, {0x0000012A, 0x00000304}, -{0x0000012B, 0x00000069}, {0x0000012B, 0x00000304}, {0x0000012C, 0x00000049}, {0x0000012C, 0x00000306}, -{0x0000012D, 0x00000069}, {0x0000012D, 0x00000306}, {0x0000012E, 0x00000049}, {0x0000012E, 0x00000328}, -{0x0000012F, 0x00000069}, {0x0000012F, 0x00000328}, {0x00000130, 0x00000049}, {0x00000130, 0x00000307}, -{0x00000134, 0x0000004A}, {0x00000134, 0x00000302}, {0x00000135, 0x0000006A}, {0x00000135, 0x00000302}, -{0x00000136, 0x0000004B}, {0x00000136, 0x00000327}, {0x00000137, 0x0000006B}, {0x00000137, 0x00000327}, -{0x00000139, 0x0000004C}, {0x00000139, 0x00000301}, {0x0000013A, 0x0000006C}, {0x0000013A, 0x00000301}, -{0x0000013B, 0x0000004C}, {0x0000013B, 0x00000327}, {0x0000013C, 0x0000006C}, {0x0000013C, 0x00000327}, -{0x0000013D, 0x0000004C}, {0x0000013D, 0x0000030C}, {0x0000013E, 0x0000006C}, {0x0000013E, 0x0000030C}, -{0x00000143, 0x0000004E}, {0x00000143, 0x00000301}, {0x00000144, 0x0000006E}, {0x00000144, 0x00000301}, -{0x00000145, 0x0000004E}, {0x00000145, 0x00000327}, {0x00000146, 0x0000006E}, {0x00000146, 0x00000327}, -{0x00000147, 0x0000004E}, {0x00000147, 0x0000030C}, {0x00000148, 0x0000006E}, {0x00000148, 0x0000030C}, -{0x0000014C, 0x0000004F}, {0x0000014C, 0x00000304}, {0x0000014D, 0x0000006F}, {0x0000014D, 0x00000304}, -{0x0000014E, 0x0000004F}, {0x0000014E, 0x00000306}, {0x0000014F, 0x0000006F}, {0x0000014F, 0x00000306}, -{0x00000150, 0x0000004F}, {0x00000150, 0x0000030B}, {0x00000151, 0x0000006F}, {0x00000151, 0x0000030B}, -{0x00000154, 0x00000052}, {0x00000154, 0x00000301}, {0x00000155, 0x00000072}, {0x00000155, 0x00000301}, -{0x00000156, 0x00000052}, {0x00000156, 0x00000327}, {0x00000157, 0x00000072}, {0x00000157, 0x00000327}, -{0x00000158, 0x00000052}, {0x00000158, 0x0000030C}, {0x00000159, 0x00000072}, {0x00000159, 0x0000030C}, -{0x0000015A, 0x00000053}, {0x0000015A, 0x00000301}, {0x0000015B, 0x00000073}, {0x0000015B, 0x00000301}, -{0x0000015C, 0x00000053}, {0x0000015C, 0x00000302}, {0x0000015D, 0x00000073}, {0x0000015D, 0x00000302}, -{0x0000015E, 0x00000053}, {0x0000015E, 0x00000327}, {0x0000015F, 0x00000073}, {0x0000015F, 0x00000327}, -{0x00000160, 0x00000053}, {0x00000160, 0x0000030C}, {0x00000161, 0x00000073}, {0x00000161, 0x0000030C}, -{0x00000162, 0x00000054}, {0x00000162, 0x00000327}, {0x00000163, 0x00000074}, {0x00000163, 0x00000327}, -{0x00000164, 0x00000054}, {0x00000164, 0x0000030C}, {0x00000165, 0x00000074}, {0x00000165, 0x0000030C}, -{0x00000168, 0x00000055}, {0x00000168, 0x00000303}, {0x00000169, 0x00000075}, {0x00000169, 0x00000303}, -{0x0000016A, 0x00000055}, {0x0000016A, 0x00000304}, {0x0000016B, 0x00000075}, {0x0000016B, 0x00000304}, -{0x0000016C, 0x00000055}, {0x0000016C, 0x00000306}, {0x0000016D, 0x00000075}, {0x0000016D, 0x00000306}, -{0x0000016E, 0x00000055}, {0x0000016E, 0x0000030A}, {0x0000016F, 0x00000075}, {0x0000016F, 0x0000030A}, -{0x00000170, 0x00000055}, {0x00000170, 0x0000030B}, {0x00000171, 0x00000075}, {0x00000171, 0x0000030B}, -{0x00000172, 0x00000055}, {0x00000172, 0x00000328}, {0x00000173, 0x00000075}, {0x00000173, 0x00000328}, -{0x00000174, 0x00000057}, {0x00000174, 0x00000302}, {0x00000175, 0x00000077}, {0x00000175, 0x00000302}, -{0x00000176, 0x00000059}, {0x00000176, 0x00000302}, {0x00000177, 0x00000079}, {0x00000177, 0x00000302}, -{0x00000178, 0x00000059}, {0x00000178, 0x00000308}, {0x00000179, 0x0000005A}, {0x00000179, 0x00000301}, -{0x0000017A, 0x0000007A}, {0x0000017A, 0x00000301}, {0x0000017B, 0x0000005A}, {0x0000017B, 0x00000307}, -{0x0000017C, 0x0000007A}, {0x0000017C, 0x00000307}, {0x0000017D, 0x0000005A}, {0x0000017D, 0x0000030C}, -{0x0000017E, 0x0000007A}, {0x0000017E, 0x0000030C}, {0x000001A0, 0x0000004F}, {0x000001A0, 0x0000031B}, -{0x000001A1, 0x0000006F}, {0x000001A1, 0x0000031B}, {0x000001AF, 0x00000055}, {0x000001AF, 0x0000031B}, -{0x000001B0, 0x00000075}, {0x000001B0, 0x0000031B}, {0x000001CD, 0x00000041}, {0x000001CD, 0x0000030C}, -{0x000001CE, 0x00000061}, {0x000001CE, 0x0000030C}, {0x000001CF, 0x00000049}, {0x000001CF, 0x0000030C}, -{0x000001D0, 0x00000069}, {0x000001D0, 0x0000030C}, {0x000001D1, 0x0000004F}, {0x000001D1, 0x0000030C}, -{0x000001D2, 0x0000006F}, {0x000001D2, 0x0000030C}, {0x000001D3, 0x00000055}, {0x000001D3, 0x0000030C}, -{0x000001D4, 0x00000075}, {0x000001D4, 0x0000030C}, {0x000001D5, 0x00000055}, {0x000001D5, 0x00000308}, -{0x000001D5, 0x00000304}, {0x000001D6, 0x00000075}, {0x000001D6, 0x00000308}, {0x000001D6, 0x00000304}, -{0x000001D7, 0x00000055}, {0x000001D7, 0x00000308}, {0x000001D7, 0x00000301}, {0x000001D8, 0x00000075}, -{0x000001D8, 0x00000308}, {0x000001D8, 0x00000301}, {0x000001D9, 0x00000055}, {0x000001D9, 0x00000308}, -{0x000001D9, 0x0000030C}, {0x000001DA, 0x00000075}, {0x000001DA, 0x00000308}, {0x000001DA, 0x0000030C}, -{0x000001DB, 0x00000055}, {0x000001DB, 0x00000308}, {0x000001DB, 0x00000300}, {0x000001DC, 0x00000075}, -{0x000001DC, 0x00000308}, {0x000001DC, 0x00000300}, {0x000001DE, 0x00000041}, {0x000001DE, 0x00000308}, -{0x000001DE, 0x00000304}, {0x000001DF, 0x00000061}, {0x000001DF, 0x00000308}, {0x000001DF, 0x00000304}, -{0x000001E0, 0x00000041}, {0x000001E0, 0x00000307}, {0x000001E0, 0x00000304}, {0x000001E1, 0x00000061}, -{0x000001E1, 0x00000307}, {0x000001E1, 0x00000304}, {0x000001E2, 0x000000C6}, {0x000001E2, 0x00000304}, -{0x000001E3, 0x000000E6}, {0x000001E3, 0x00000304}, {0x000001E6, 0x00000047}, {0x000001E6, 0x0000030C}, -{0x000001E7, 0x00000067}, {0x000001E7, 0x0000030C}, {0x000001E8, 0x0000004B}, {0x000001E8, 0x0000030C}, -{0x000001E9, 0x0000006B}, {0x000001E9, 0x0000030C}, {0x000001EA, 0x0000004F}, {0x000001EA, 0x00000328}, -{0x000001EB, 0x0000006F}, {0x000001EB, 0x00000328}, {0x000001EC, 0x0000004F}, {0x000001EC, 0x00000328}, -{0x000001EC, 0x00000304}, {0x000001ED, 0x0000006F}, {0x000001ED, 0x00000328}, {0x000001ED, 0x00000304}, -{0x000001EE, 0x000001B7}, {0x000001EE, 0x0000030C}, {0x000001EF, 0x00000292}, {0x000001EF, 0x0000030C}, -{0x000001F0, 0x0000006A}, {0x000001F0, 0x0000030C}, {0x000001F4, 0x00000047}, {0x000001F4, 0x00000301}, -{0x000001F5, 0x00000067}, {0x000001F5, 0x00000301}, {0x000001F8, 0x0000004E}, {0x000001F8, 0x00000300}, -{0x000001F9, 0x0000006E}, {0x000001F9, 0x00000300}, {0x000001FA, 0x00000041}, {0x000001FA, 0x0000030A}, -{0x000001FA, 0x00000301}, {0x000001FB, 0x00000061}, {0x000001FB, 0x0000030A}, {0x000001FB, 0x00000301}, -{0x000001FC, 0x000000C6}, {0x000001FC, 0x00000301}, {0x000001FD, 0x000000E6}, {0x000001FD, 0x00000301}, -{0x000001FE, 0x000000D8}, {0x000001FE, 0x00000301}, {0x000001FF, 0x000000F8}, {0x000001FF, 0x00000301}, -{0x00000200, 0x00000041}, {0x00000200, 0x0000030F}, {0x00000201, 0x00000061}, {0x00000201, 0x0000030F}, -{0x00000202, 0x00000041}, {0x00000202, 0x00000311}, {0x00000203, 0x00000061}, {0x00000203, 0x00000311}, -{0x00000204, 0x00000045}, {0x00000204, 0x0000030F}, {0x00000205, 0x00000065}, {0x00000205, 0x0000030F}, -{0x00000206, 0x00000045}, {0x00000206, 0x00000311}, {0x00000207, 0x00000065}, {0x00000207, 0x00000311}, -{0x00000208, 0x00000049}, {0x00000208, 0x0000030F}, {0x00000209, 0x00000069}, {0x00000209, 0x0000030F}, -{0x0000020A, 0x00000049}, {0x0000020A, 0x00000311}, {0x0000020B, 0x00000069}, {0x0000020B, 0x00000311}, -{0x0000020C, 0x0000004F}, {0x0000020C, 0x0000030F}, {0x0000020D, 0x0000006F}, {0x0000020D, 0x0000030F}, -{0x0000020E, 0x0000004F}, {0x0000020E, 0x00000311}, {0x0000020F, 0x0000006F}, {0x0000020F, 0x00000311}, -{0x00000210, 0x00000052}, {0x00000210, 0x0000030F}, {0x00000211, 0x00000072}, {0x00000211, 0x0000030F}, -{0x00000212, 0x00000052}, {0x00000212, 0x00000311}, {0x00000213, 0x00000072}, {0x00000213, 0x00000311}, -{0x00000214, 0x00000055}, {0x00000214, 0x0000030F}, {0x00000215, 0x00000075}, {0x00000215, 0x0000030F}, -{0x00000216, 0x00000055}, {0x00000216, 0x00000311}, {0x00000217, 0x00000075}, {0x00000217, 0x00000311}, -{0x00000218, 0x00000053}, {0x00000218, 0x00000326}, {0x00000219, 0x00000073}, {0x00000219, 0x00000326}, -{0x0000021A, 0x00000054}, {0x0000021A, 0x00000326}, {0x0000021B, 0x00000074}, {0x0000021B, 0x00000326}, -{0x0000021E, 0x00000048}, {0x0000021E, 0x0000030C}, {0x0000021F, 0x00000068}, {0x0000021F, 0x0000030C}, -{0x00000226, 0x00000041}, {0x00000226, 0x00000307}, {0x00000227, 0x00000061}, {0x00000227, 0x00000307}, -{0x00000228, 0x00000045}, {0x00000228, 0x00000327}, {0x00000229, 0x00000065}, {0x00000229, 0x00000327}, -{0x0000022A, 0x0000004F}, {0x0000022A, 0x00000308}, {0x0000022A, 0x00000304}, {0x0000022B, 0x0000006F}, -{0x0000022B, 0x00000308}, {0x0000022B, 0x00000304}, {0x0000022C, 0x0000004F}, {0x0000022C, 0x00000303}, -{0x0000022C, 0x00000304}, {0x0000022D, 0x0000006F}, {0x0000022D, 0x00000303}, {0x0000022D, 0x00000304}, -{0x0000022E, 0x0000004F}, {0x0000022E, 0x00000307}, {0x0000022F, 0x0000006F}, {0x0000022F, 0x00000307}, -{0x00000230, 0x0000004F}, {0x00000230, 0x00000307}, {0x00000230, 0x00000304}, {0x00000231, 0x0000006F}, -{0x00000231, 0x00000307}, {0x00000231, 0x00000304}, {0x00000232, 0x00000059}, {0x00000232, 0x00000304}, -{0x00000233, 0x00000079}, {0x00000233, 0x00000304}, {0x00000340, 0x00000300}, {0x00000341, 0x00000301}, -{0x00000343, 0x00000313}, {0x00000344, 0x00000308}, {0x00000344, 0x00000301}, {0x00000374, 0x000002B9}, -{0x0000037E, 0x0000003B}, {0x00000385, 0x000000A8}, {0x00000385, 0x00000301}, {0x00000386, 0x00000391}, -{0x00000386, 0x00000301}, {0x00000387, 0x000000B7}, {0x00000388, 0x00000395}, {0x00000388, 0x00000301}, -{0x00000389, 0x00000397}, {0x00000389, 0x00000301}, {0x0000038A, 0x00000399}, {0x0000038A, 0x00000301}, -{0x0000038C, 0x0000039F}, {0x0000038C, 0x00000301}, {0x0000038E, 0x000003A5}, {0x0000038E, 0x00000301}, -{0x0000038F, 0x000003A9}, {0x0000038F, 0x00000301}, {0x00000390, 0x000003B9}, {0x00000390, 0x00000308}, -{0x00000390, 0x00000301}, {0x000003AA, 0x00000399}, {0x000003AA, 0x00000308}, {0x000003AB, 0x000003A5}, -{0x000003AB, 0x00000308}, {0x000003AC, 0x000003B1}, {0x000003AC, 0x00000301}, {0x000003AD, 0x000003B5}, -{0x000003AD, 0x00000301}, {0x000003AE, 0x000003B7}, {0x000003AE, 0x00000301}, {0x000003AF, 0x000003B9}, -{0x000003AF, 0x00000301}, {0x000003B0, 0x000003C5}, {0x000003B0, 0x00000308}, {0x000003B0, 0x00000301}, -{0x000003CA, 0x000003B9}, {0x000003CA, 0x00000308}, {0x000003CB, 0x000003C5}, {0x000003CB, 0x00000308}, -{0x000003CC, 0x000003BF}, {0x000003CC, 0x00000301}, {0x000003CD, 0x000003C5}, {0x000003CD, 0x00000301}, -{0x000003CE, 0x000003C9}, {0x000003CE, 0x00000301}, {0x000003D3, 0x000003D2}, {0x000003D3, 0x00000301}, -{0x000003D4, 0x000003D2}, {0x000003D4, 0x00000308}, {0x00000400, 0x00000415}, {0x00000400, 0x00000300}, -{0x00000401, 0x00000415}, {0x00000401, 0x00000308}, {0x00000403, 0x00000413}, {0x00000403, 0x00000301}, -{0x00000407, 0x00000406}, {0x00000407, 0x00000308}, {0x0000040C, 0x0000041A}, {0x0000040C, 0x00000301}, -{0x0000040D, 0x00000418}, {0x0000040D, 0x00000300}, {0x0000040E, 0x00000423}, {0x0000040E, 0x00000306}, -{0x00000419, 0x00000418}, {0x00000419, 0x00000306}, {0x00000439, 0x00000438}, {0x00000439, 0x00000306}, -{0x00000450, 0x00000435}, {0x00000450, 0x00000300}, {0x00000451, 0x00000435}, {0x00000451, 0x00000308}, -{0x00000453, 0x00000433}, {0x00000453, 0x00000301}, {0x00000457, 0x00000456}, {0x00000457, 0x00000308}, -{0x0000045C, 0x0000043A}, {0x0000045C, 0x00000301}, {0x0000045D, 0x00000438}, {0x0000045D, 0x00000300}, -{0x0000045E, 0x00000443}, {0x0000045E, 0x00000306}, {0x00000476, 0x00000474}, {0x00000476, 0x0000030F}, -{0x00000477, 0x00000475}, {0x00000477, 0x0000030F}, {0x000004C1, 0x00000416}, {0x000004C1, 0x00000306}, -{0x000004C2, 0x00000436}, {0x000004C2, 0x00000306}, {0x000004D0, 0x00000410}, {0x000004D0, 0x00000306}, -{0x000004D1, 0x00000430}, {0x000004D1, 0x00000306}, {0x000004D2, 0x00000410}, {0x000004D2, 0x00000308}, -{0x000004D3, 0x00000430}, {0x000004D3, 0x00000308}, {0x000004D6, 0x00000415}, {0x000004D6, 0x00000306}, -{0x000004D7, 0x00000435}, {0x000004D7, 0x00000306}, {0x000004DA, 0x000004D8}, {0x000004DA, 0x00000308}, -{0x000004DB, 0x000004D9}, {0x000004DB, 0x00000308}, {0x000004DC, 0x00000416}, {0x000004DC, 0x00000308}, -{0x000004DD, 0x00000436}, {0x000004DD, 0x00000308}, {0x000004DE, 0x00000417}, {0x000004DE, 0x00000308}, -{0x000004DF, 0x00000437}, {0x000004DF, 0x00000308}, {0x000004E2, 0x00000418}, {0x000004E2, 0x00000304}, -{0x000004E3, 0x00000438}, {0x000004E3, 0x00000304}, {0x000004E4, 0x00000418}, {0x000004E4, 0x00000308}, -{0x000004E5, 0x00000438}, {0x000004E5, 0x00000308}, {0x000004E6, 0x0000041E}, {0x000004E6, 0x00000308}, -{0x000004E7, 0x0000043E}, {0x000004E7, 0x00000308}, {0x000004EA, 0x000004E8}, {0x000004EA, 0x00000308}, -{0x000004EB, 0x000004E9}, {0x000004EB, 0x00000308}, {0x000004EC, 0x0000042D}, {0x000004EC, 0x00000308}, -{0x000004ED, 0x0000044D}, {0x000004ED, 0x00000308}, {0x000004EE, 0x00000423}, {0x000004EE, 0x00000304}, -{0x000004EF, 0x00000443}, {0x000004EF, 0x00000304}, {0x000004F0, 0x00000423}, {0x000004F0, 0x00000308}, -{0x000004F1, 0x00000443}, {0x000004F1, 0x00000308}, {0x000004F2, 0x00000423}, {0x000004F2, 0x0000030B}, -{0x000004F3, 0x00000443}, {0x000004F3, 0x0000030B}, {0x000004F4, 0x00000427}, {0x000004F4, 0x00000308}, -{0x000004F5, 0x00000447}, {0x000004F5, 0x00000308}, {0x000004F8, 0x0000042B}, {0x000004F8, 0x00000308}, -{0x000004F9, 0x0000044B}, {0x000004F9, 0x00000308}, {0x00000622, 0x00000627}, {0x00000622, 0x00000653}, -{0x00000623, 0x00000627}, {0x00000623, 0x00000654}, {0x00000624, 0x00000648}, {0x00000624, 0x00000654}, -{0x00000625, 0x00000627}, {0x00000625, 0x00000655}, {0x00000626, 0x0000064A}, {0x00000626, 0x00000654}, -{0x000006C0, 0x000006D5}, {0x000006C0, 0x00000654}, {0x000006C2, 0x000006C1}, {0x000006C2, 0x00000654}, -{0x000006D3, 0x000006D2}, {0x000006D3, 0x00000654}, {0x00000929, 0x00000928}, {0x00000929, 0x0000093C}, -{0x00000931, 0x00000930}, {0x00000931, 0x0000093C}, {0x00000934, 0x00000933}, {0x00000934, 0x0000093C}, -{0x00000958, 0x00000915}, {0x00000958, 0x0000093C}, {0x00000959, 0x00000916}, {0x00000959, 0x0000093C}, -{0x0000095A, 0x00000917}, {0x0000095A, 0x0000093C}, {0x0000095B, 0x0000091C}, {0x0000095B, 0x0000093C}, -{0x0000095C, 0x00000921}, {0x0000095C, 0x0000093C}, {0x0000095D, 0x00000922}, {0x0000095D, 0x0000093C}, -{0x0000095E, 0x0000092B}, {0x0000095E, 0x0000093C}, {0x0000095F, 0x0000092F}, {0x0000095F, 0x0000093C}, -{0x000009CB, 0x000009C7}, {0x000009CB, 0x000009BE}, {0x000009CC, 0x000009C7}, {0x000009CC, 0x000009D7}, -{0x000009DC, 0x000009A1}, {0x000009DC, 0x000009BC}, {0x000009DD, 0x000009A2}, {0x000009DD, 0x000009BC}, -{0x000009DF, 0x000009AF}, {0x000009DF, 0x000009BC}, {0x00000A33, 0x00000A32}, {0x00000A33, 0x00000A3C}, -{0x00000A36, 0x00000A38}, {0x00000A36, 0x00000A3C}, {0x00000A59, 0x00000A16}, {0x00000A59, 0x00000A3C}, -{0x00000A5A, 0x00000A17}, {0x00000A5A, 0x00000A3C}, {0x00000A5B, 0x00000A1C}, {0x00000A5B, 0x00000A3C}, -{0x00000A5E, 0x00000A2B}, {0x00000A5E, 0x00000A3C}, {0x00000B48, 0x00000B47}, {0x00000B48, 0x00000B56}, -{0x00000B4B, 0x00000B47}, {0x00000B4B, 0x00000B3E}, {0x00000B4C, 0x00000B47}, {0x00000B4C, 0x00000B57}, -{0x00000B5C, 0x00000B21}, {0x00000B5C, 0x00000B3C}, {0x00000B5D, 0x00000B22}, {0x00000B5D, 0x00000B3C}, -{0x00000B94, 0x00000B92}, {0x00000B94, 0x00000BD7}, {0x00000BCA, 0x00000BC6}, {0x00000BCA, 0x00000BBE}, -{0x00000BCB, 0x00000BC7}, {0x00000BCB, 0x00000BBE}, {0x00000BCC, 0x00000BC6}, {0x00000BCC, 0x00000BD7}, -{0x00000C48, 0x00000C46}, {0x00000C48, 0x00000C56}, {0x00000CC0, 0x00000CBF}, {0x00000CC0, 0x00000CD5}, -{0x00000CC7, 0x00000CC6}, {0x00000CC7, 0x00000CD5}, {0x00000CC8, 0x00000CC6}, {0x00000CC8, 0x00000CD6}, -{0x00000CCA, 0x00000CC6}, {0x00000CCA, 0x00000CC2}, {0x00000CCB, 0x00000CC6}, {0x00000CCB, 0x00000CC2}, -{0x00000CCB, 0x00000CD5}, {0x00000D4A, 0x00000D46}, {0x00000D4A, 0x00000D3E}, {0x00000D4B, 0x00000D47}, -{0x00000D4B, 0x00000D3E}, {0x00000D4C, 0x00000D46}, {0x00000D4C, 0x00000D57}, {0x00000DDA, 0x00000DD9}, -{0x00000DDA, 0x00000DCA}, {0x00000DDC, 0x00000DD9}, {0x00000DDC, 0x00000DCF}, {0x00000DDD, 0x00000DD9}, -{0x00000DDD, 0x00000DCF}, {0x00000DDD, 0x00000DCA}, {0x00000DDE, 0x00000DD9}, {0x00000DDE, 0x00000DDF}, -{0x00000F43, 0x00000F42}, {0x00000F43, 0x00000FB7}, {0x00000F4D, 0x00000F4C}, {0x00000F4D, 0x00000FB7}, -{0x00000F52, 0x00000F51}, {0x00000F52, 0x00000FB7}, {0x00000F57, 0x00000F56}, {0x00000F57, 0x00000FB7}, -{0x00000F5C, 0x00000F5B}, {0x00000F5C, 0x00000FB7}, {0x00000F69, 0x00000F40}, {0x00000F69, 0x00000FB5}, -{0x00000F73, 0x00000F71}, {0x00000F73, 0x00000F72}, {0x00000F75, 0x00000F71}, {0x00000F75, 0x00000F74}, -{0x00000F76, 0x00000FB2}, {0x00000F76, 0x00000F80}, {0x00000F78, 0x00000FB3}, {0x00000F78, 0x00000F80}, -{0x00000F81, 0x00000F71}, {0x00000F81, 0x00000F80}, {0x00000F93, 0x00000F92}, {0x00000F93, 0x00000FB7}, -{0x00000F9D, 0x00000F9C}, {0x00000F9D, 0x00000FB7}, {0x00000FA2, 0x00000FA1}, {0x00000FA2, 0x00000FB7}, -{0x00000FA7, 0x00000FA6}, {0x00000FA7, 0x00000FB7}, {0x00000FAC, 0x00000FAB}, {0x00000FAC, 0x00000FB7}, -{0x00000FB9, 0x00000F90}, {0x00000FB9, 0x00000FB5}, {0x00001026, 0x00001025}, {0x00001026, 0x0000102E}, -{0x00001B06, 0x00001B05}, {0x00001B06, 0x00001B35}, {0x00001B08, 0x00001B07}, {0x00001B08, 0x00001B35}, -{0x00001B0A, 0x00001B09}, {0x00001B0A, 0x00001B35}, {0x00001B0C, 0x00001B0B}, {0x00001B0C, 0x00001B35}, -{0x00001B0E, 0x00001B0D}, {0x00001B0E, 0x00001B35}, {0x00001B12, 0x00001B11}, {0x00001B12, 0x00001B35}, -{0x00001B3B, 0x00001B3A}, {0x00001B3B, 0x00001B35}, {0x00001B3D, 0x00001B3C}, {0x00001B3D, 0x00001B35}, -{0x00001B40, 0x00001B3E}, {0x00001B40, 0x00001B35}, {0x00001B41, 0x00001B3F}, {0x00001B41, 0x00001B35}, -{0x00001B43, 0x00001B42}, {0x00001B43, 0x00001B35}, {0x00001E00, 0x00000041}, {0x00001E00, 0x00000325}, -{0x00001E01, 0x00000061}, {0x00001E01, 0x00000325}, {0x00001E02, 0x00000042}, {0x00001E02, 0x00000307}, -{0x00001E03, 0x00000062}, {0x00001E03, 0x00000307}, {0x00001E04, 0x00000042}, {0x00001E04, 0x00000323}, -{0x00001E05, 0x00000062}, {0x00001E05, 0x00000323}, {0x00001E06, 0x00000042}, {0x00001E06, 0x00000331}, -{0x00001E07, 0x00000062}, {0x00001E07, 0x00000331}, {0x00001E08, 0x00000043}, {0x00001E08, 0x00000327}, -{0x00001E08, 0x00000301}, {0x00001E09, 0x00000063}, {0x00001E09, 0x00000327}, {0x00001E09, 0x00000301}, -{0x00001E0A, 0x00000044}, {0x00001E0A, 0x00000307}, {0x00001E0B, 0x00000064}, {0x00001E0B, 0x00000307}, -{0x00001E0C, 0x00000044}, {0x00001E0C, 0x00000323}, {0x00001E0D, 0x00000064}, {0x00001E0D, 0x00000323}, -{0x00001E0E, 0x00000044}, {0x00001E0E, 0x00000331}, {0x00001E0F, 0x00000064}, {0x00001E0F, 0x00000331}, -{0x00001E10, 0x00000044}, {0x00001E10, 0x00000327}, {0x00001E11, 0x00000064}, {0x00001E11, 0x00000327}, -{0x00001E12, 0x00000044}, {0x00001E12, 0x0000032D}, {0x00001E13, 0x00000064}, {0x00001E13, 0x0000032D}, -{0x00001E14, 0x00000045}, {0x00001E14, 0x00000304}, {0x00001E14, 0x00000300}, {0x00001E15, 0x00000065}, -{0x00001E15, 0x00000304}, {0x00001E15, 0x00000300}, {0x00001E16, 0x00000045}, {0x00001E16, 0x00000304}, -{0x00001E16, 0x00000301}, {0x00001E17, 0x00000065}, {0x00001E17, 0x00000304}, {0x00001E17, 0x00000301}, -{0x00001E18, 0x00000045}, {0x00001E18, 0x0000032D}, {0x00001E19, 0x00000065}, {0x00001E19, 0x0000032D}, -{0x00001E1A, 0x00000045}, {0x00001E1A, 0x00000330}, {0x00001E1B, 0x00000065}, {0x00001E1B, 0x00000330}, -{0x00001E1C, 0x00000045}, {0x00001E1C, 0x00000327}, {0x00001E1C, 0x00000306}, {0x00001E1D, 0x00000065}, -{0x00001E1D, 0x00000327}, {0x00001E1D, 0x00000306}, {0x00001E1E, 0x00000046}, {0x00001E1E, 0x00000307}, -{0x00001E1F, 0x00000066}, {0x00001E1F, 0x00000307}, {0x00001E20, 0x00000047}, {0x00001E20, 0x00000304}, -{0x00001E21, 0x00000067}, {0x00001E21, 0x00000304}, {0x00001E22, 0x00000048}, {0x00001E22, 0x00000307}, -{0x00001E23, 0x00000068}, {0x00001E23, 0x00000307}, {0x00001E24, 0x00000048}, {0x00001E24, 0x00000323}, -{0x00001E25, 0x00000068}, {0x00001E25, 0x00000323}, {0x00001E26, 0x00000048}, {0x00001E26, 0x00000308}, -{0x00001E27, 0x00000068}, {0x00001E27, 0x00000308}, {0x00001E28, 0x00000048}, {0x00001E28, 0x00000327}, -{0x00001E29, 0x00000068}, {0x00001E29, 0x00000327}, {0x00001E2A, 0x00000048}, {0x00001E2A, 0x0000032E}, -{0x00001E2B, 0x00000068}, {0x00001E2B, 0x0000032E}, {0x00001E2C, 0x00000049}, {0x00001E2C, 0x00000330}, -{0x00001E2D, 0x00000069}, {0x00001E2D, 0x00000330}, {0x00001E2E, 0x00000049}, {0x00001E2E, 0x00000308}, -{0x00001E2E, 0x00000301}, {0x00001E2F, 0x00000069}, {0x00001E2F, 0x00000308}, {0x00001E2F, 0x00000301}, -{0x00001E30, 0x0000004B}, {0x00001E30, 0x00000301}, {0x00001E31, 0x0000006B}, {0x00001E31, 0x00000301}, -{0x00001E32, 0x0000004B}, {0x00001E32, 0x00000323}, {0x00001E33, 0x0000006B}, {0x00001E33, 0x00000323}, -{0x00001E34, 0x0000004B}, {0x00001E34, 0x00000331}, {0x00001E35, 0x0000006B}, {0x00001E35, 0x00000331}, -{0x00001E36, 0x0000004C}, {0x00001E36, 0x00000323}, {0x00001E37, 0x0000006C}, {0x00001E37, 0x00000323}, -{0x00001E38, 0x0000004C}, {0x00001E38, 0x00000323}, {0x00001E38, 0x00000304}, {0x00001E39, 0x0000006C}, -{0x00001E39, 0x00000323}, {0x00001E39, 0x00000304}, {0x00001E3A, 0x0000004C}, {0x00001E3A, 0x00000331}, -{0x00001E3B, 0x0000006C}, {0x00001E3B, 0x00000331}, {0x00001E3C, 0x0000004C}, {0x00001E3C, 0x0000032D}, -{0x00001E3D, 0x0000006C}, {0x00001E3D, 0x0000032D}, {0x00001E3E, 0x0000004D}, {0x00001E3E, 0x00000301}, -{0x00001E3F, 0x0000006D}, {0x00001E3F, 0x00000301}, {0x00001E40, 0x0000004D}, {0x00001E40, 0x00000307}, -{0x00001E41, 0x0000006D}, {0x00001E41, 0x00000307}, {0x00001E42, 0x0000004D}, {0x00001E42, 0x00000323}, -{0x00001E43, 0x0000006D}, {0x00001E43, 0x00000323}, {0x00001E44, 0x0000004E}, {0x00001E44, 0x00000307}, -{0x00001E45, 0x0000006E}, {0x00001E45, 0x00000307}, {0x00001E46, 0x0000004E}, {0x00001E46, 0x00000323}, -{0x00001E47, 0x0000006E}, {0x00001E47, 0x00000323}, {0x00001E48, 0x0000004E}, {0x00001E48, 0x00000331}, -{0x00001E49, 0x0000006E}, {0x00001E49, 0x00000331}, {0x00001E4A, 0x0000004E}, {0x00001E4A, 0x0000032D}, -{0x00001E4B, 0x0000006E}, {0x00001E4B, 0x0000032D}, {0x00001E4C, 0x0000004F}, {0x00001E4C, 0x00000303}, -{0x00001E4C, 0x00000301}, {0x00001E4D, 0x0000006F}, {0x00001E4D, 0x00000303}, {0x00001E4D, 0x00000301}, -{0x00001E4E, 0x0000004F}, {0x00001E4E, 0x00000303}, {0x00001E4E, 0x00000308}, {0x00001E4F, 0x0000006F}, -{0x00001E4F, 0x00000303}, {0x00001E4F, 0x00000308}, {0x00001E50, 0x0000004F}, {0x00001E50, 0x00000304}, -{0x00001E50, 0x00000300}, {0x00001E51, 0x0000006F}, {0x00001E51, 0x00000304}, {0x00001E51, 0x00000300}, -{0x00001E52, 0x0000004F}, {0x00001E52, 0x00000304}, {0x00001E52, 0x00000301}, {0x00001E53, 0x0000006F}, -{0x00001E53, 0x00000304}, {0x00001E53, 0x00000301}, {0x00001E54, 0x00000050}, {0x00001E54, 0x00000301}, -{0x00001E55, 0x00000070}, {0x00001E55, 0x00000301}, {0x00001E56, 0x00000050}, {0x00001E56, 0x00000307}, -{0x00001E57, 0x00000070}, {0x00001E57, 0x00000307}, {0x00001E58, 0x00000052}, {0x00001E58, 0x00000307}, -{0x00001E59, 0x00000072}, {0x00001E59, 0x00000307}, {0x00001E5A, 0x00000052}, {0x00001E5A, 0x00000323}, -{0x00001E5B, 0x00000072}, {0x00001E5B, 0x00000323}, {0x00001E5C, 0x00000052}, {0x00001E5C, 0x00000323}, -{0x00001E5C, 0x00000304}, {0x00001E5D, 0x00000072}, {0x00001E5D, 0x00000323}, {0x00001E5D, 0x00000304}, -{0x00001E5E, 0x00000052}, {0x00001E5E, 0x00000331}, {0x00001E5F, 0x00000072}, {0x00001E5F, 0x00000331}, -{0x00001E60, 0x00000053}, {0x00001E60, 0x00000307}, {0x00001E61, 0x00000073}, {0x00001E61, 0x00000307}, -{0x00001E62, 0x00000053}, {0x00001E62, 0x00000323}, {0x00001E63, 0x00000073}, {0x00001E63, 0x00000323}, -{0x00001E64, 0x00000053}, {0x00001E64, 0x00000301}, {0x00001E64, 0x00000307}, {0x00001E65, 0x00000073}, -{0x00001E65, 0x00000301}, {0x00001E65, 0x00000307}, {0x00001E66, 0x00000053}, {0x00001E66, 0x0000030C}, -{0x00001E66, 0x00000307}, {0x00001E67, 0x00000073}, {0x00001E67, 0x0000030C}, {0x00001E67, 0x00000307}, -{0x00001E68, 0x00000053}, {0x00001E68, 0x00000323}, {0x00001E68, 0x00000307}, {0x00001E69, 0x00000073}, -{0x00001E69, 0x00000323}, {0x00001E69, 0x00000307}, {0x00001E6A, 0x00000054}, {0x00001E6A, 0x00000307}, -{0x00001E6B, 0x00000074}, {0x00001E6B, 0x00000307}, {0x00001E6C, 0x00000054}, {0x00001E6C, 0x00000323}, -{0x00001E6D, 0x00000074}, {0x00001E6D, 0x00000323}, {0x00001E6E, 0x00000054}, {0x00001E6E, 0x00000331}, -{0x00001E6F, 0x00000074}, {0x00001E6F, 0x00000331}, {0x00001E70, 0x00000054}, {0x00001E70, 0x0000032D}, -{0x00001E71, 0x00000074}, {0x00001E71, 0x0000032D}, {0x00001E72, 0x00000055}, {0x00001E72, 0x00000324}, -{0x00001E73, 0x00000075}, {0x00001E73, 0x00000324}, {0x00001E74, 0x00000055}, {0x00001E74, 0x00000330}, -{0x00001E75, 0x00000075}, {0x00001E75, 0x00000330}, {0x00001E76, 0x00000055}, {0x00001E76, 0x0000032D}, -{0x00001E77, 0x00000075}, {0x00001E77, 0x0000032D}, {0x00001E78, 0x00000055}, {0x00001E78, 0x00000303}, -{0x00001E78, 0x00000301}, {0x00001E79, 0x00000075}, {0x00001E79, 0x00000303}, {0x00001E79, 0x00000301}, -{0x00001E7A, 0x00000055}, {0x00001E7A, 0x00000304}, {0x00001E7A, 0x00000308}, {0x00001E7B, 0x00000075}, -{0x00001E7B, 0x00000304}, {0x00001E7B, 0x00000308}, {0x00001E7C, 0x00000056}, {0x00001E7C, 0x00000303}, -{0x00001E7D, 0x00000076}, {0x00001E7D, 0x00000303}, {0x00001E7E, 0x00000056}, {0x00001E7E, 0x00000323}, -{0x00001E7F, 0x00000076}, {0x00001E7F, 0x00000323}, {0x00001E80, 0x00000057}, {0x00001E80, 0x00000300}, -{0x00001E81, 0x00000077}, {0x00001E81, 0x00000300}, {0x00001E82, 0x00000057}, {0x00001E82, 0x00000301}, -{0x00001E83, 0x00000077}, {0x00001E83, 0x00000301}, {0x00001E84, 0x00000057}, {0x00001E84, 0x00000308}, -{0x00001E85, 0x00000077}, {0x00001E85, 0x00000308}, {0x00001E86, 0x00000057}, {0x00001E86, 0x00000307}, -{0x00001E87, 0x00000077}, {0x00001E87, 0x00000307}, {0x00001E88, 0x00000057}, {0x00001E88, 0x00000323}, -{0x00001E89, 0x00000077}, {0x00001E89, 0x00000323}, {0x00001E8A, 0x00000058}, {0x00001E8A, 0x00000307}, -{0x00001E8B, 0x00000078}, {0x00001E8B, 0x00000307}, {0x00001E8C, 0x00000058}, {0x00001E8C, 0x00000308}, -{0x00001E8D, 0x00000078}, {0x00001E8D, 0x00000308}, {0x00001E8E, 0x00000059}, {0x00001E8E, 0x00000307}, -{0x00001E8F, 0x00000079}, {0x00001E8F, 0x00000307}, {0x00001E90, 0x0000005A}, {0x00001E90, 0x00000302}, -{0x00001E91, 0x0000007A}, {0x00001E91, 0x00000302}, {0x00001E92, 0x0000005A}, {0x00001E92, 0x00000323}, -{0x00001E93, 0x0000007A}, {0x00001E93, 0x00000323}, {0x00001E94, 0x0000005A}, {0x00001E94, 0x00000331}, -{0x00001E95, 0x0000007A}, {0x00001E95, 0x00000331}, {0x00001E96, 0x00000068}, {0x00001E96, 0x00000331}, -{0x00001E97, 0x00000074}, {0x00001E97, 0x00000308}, {0x00001E98, 0x00000077}, {0x00001E98, 0x0000030A}, -{0x00001E99, 0x00000079}, {0x00001E99, 0x0000030A}, {0x00001E9B, 0x0000017F}, {0x00001E9B, 0x00000307}, -{0x00001EA0, 0x00000041}, {0x00001EA0, 0x00000323}, {0x00001EA1, 0x00000061}, {0x00001EA1, 0x00000323}, -{0x00001EA2, 0x00000041}, {0x00001EA2, 0x00000309}, {0x00001EA3, 0x00000061}, {0x00001EA3, 0x00000309}, -{0x00001EA4, 0x00000041}, {0x00001EA4, 0x00000302}, {0x00001EA4, 0x00000301}, {0x00001EA5, 0x00000061}, -{0x00001EA5, 0x00000302}, {0x00001EA5, 0x00000301}, {0x00001EA6, 0x00000041}, {0x00001EA6, 0x00000302}, -{0x00001EA6, 0x00000300}, {0x00001EA7, 0x00000061}, {0x00001EA7, 0x00000302}, {0x00001EA7, 0x00000300}, -{0x00001EA8, 0x00000041}, {0x00001EA8, 0x00000302}, {0x00001EA8, 0x00000309}, {0x00001EA9, 0x00000061}, -{0x00001EA9, 0x00000302}, {0x00001EA9, 0x00000309}, {0x00001EAA, 0x00000041}, {0x00001EAA, 0x00000302}, -{0x00001EAA, 0x00000303}, {0x00001EAB, 0x00000061}, {0x00001EAB, 0x00000302}, {0x00001EAB, 0x00000303}, -{0x00001EAC, 0x00000041}, {0x00001EAC, 0x00000323}, {0x00001EAC, 0x00000302}, {0x00001EAD, 0x00000061}, -{0x00001EAD, 0x00000323}, {0x00001EAD, 0x00000302}, {0x00001EAE, 0x00000041}, {0x00001EAE, 0x00000306}, -{0x00001EAE, 0x00000301}, {0x00001EAF, 0x00000061}, {0x00001EAF, 0x00000306}, {0x00001EAF, 0x00000301}, -{0x00001EB0, 0x00000041}, {0x00001EB0, 0x00000306}, {0x00001EB0, 0x00000300}, {0x00001EB1, 0x00000061}, -{0x00001EB1, 0x00000306}, {0x00001EB1, 0x00000300}, {0x00001EB2, 0x00000041}, {0x00001EB2, 0x00000306}, -{0x00001EB2, 0x00000309}, {0x00001EB3, 0x00000061}, {0x00001EB3, 0x00000306}, {0x00001EB3, 0x00000309}, -{0x00001EB4, 0x00000041}, {0x00001EB4, 0x00000306}, {0x00001EB4, 0x00000303}, {0x00001EB5, 0x00000061}, -{0x00001EB5, 0x00000306}, {0x00001EB5, 0x00000303}, {0x00001EB6, 0x00000041}, {0x00001EB6, 0x00000323}, -{0x00001EB6, 0x00000306}, {0x00001EB7, 0x00000061}, {0x00001EB7, 0x00000323}, {0x00001EB7, 0x00000306}, -{0x00001EB8, 0x00000045}, {0x00001EB8, 0x00000323}, {0x00001EB9, 0x00000065}, {0x00001EB9, 0x00000323}, -{0x00001EBA, 0x00000045}, {0x00001EBA, 0x00000309}, {0x00001EBB, 0x00000065}, {0x00001EBB, 0x00000309}, -{0x00001EBC, 0x00000045}, {0x00001EBC, 0x00000303}, {0x00001EBD, 0x00000065}, {0x00001EBD, 0x00000303}, -{0x00001EBE, 0x00000045}, {0x00001EBE, 0x00000302}, {0x00001EBE, 0x00000301}, {0x00001EBF, 0x00000065}, -{0x00001EBF, 0x00000302}, {0x00001EBF, 0x00000301}, {0x00001EC0, 0x00000045}, {0x00001EC0, 0x00000302}, -{0x00001EC0, 0x00000300}, {0x00001EC1, 0x00000065}, {0x00001EC1, 0x00000302}, {0x00001EC1, 0x00000300}, -{0x00001EC2, 0x00000045}, {0x00001EC2, 0x00000302}, {0x00001EC2, 0x00000309}, {0x00001EC3, 0x00000065}, -{0x00001EC3, 0x00000302}, {0x00001EC3, 0x00000309}, {0x00001EC4, 0x00000045}, {0x00001EC4, 0x00000302}, -{0x00001EC4, 0x00000303}, {0x00001EC5, 0x00000065}, {0x00001EC5, 0x00000302}, {0x00001EC5, 0x00000303}, -{0x00001EC6, 0x00000045}, {0x00001EC6, 0x00000323}, {0x00001EC6, 0x00000302}, {0x00001EC7, 0x00000065}, -{0x00001EC7, 0x00000323}, {0x00001EC7, 0x00000302}, {0x00001EC8, 0x00000049}, {0x00001EC8, 0x00000309}, -{0x00001EC9, 0x00000069}, {0x00001EC9, 0x00000309}, {0x00001ECA, 0x00000049}, {0x00001ECA, 0x00000323}, -{0x00001ECB, 0x00000069}, {0x00001ECB, 0x00000323}, {0x00001ECC, 0x0000004F}, {0x00001ECC, 0x00000323}, -{0x00001ECD, 0x0000006F}, {0x00001ECD, 0x00000323}, {0x00001ECE, 0x0000004F}, {0x00001ECE, 0x00000309}, -{0x00001ECF, 0x0000006F}, {0x00001ECF, 0x00000309}, {0x00001ED0, 0x0000004F}, {0x00001ED0, 0x00000302}, -{0x00001ED0, 0x00000301}, {0x00001ED1, 0x0000006F}, {0x00001ED1, 0x00000302}, {0x00001ED1, 0x00000301}, -{0x00001ED2, 0x0000004F}, {0x00001ED2, 0x00000302}, {0x00001ED2, 0x00000300}, {0x00001ED3, 0x0000006F}, -{0x00001ED3, 0x00000302}, {0x00001ED3, 0x00000300}, {0x00001ED4, 0x0000004F}, {0x00001ED4, 0x00000302}, -{0x00001ED4, 0x00000309}, {0x00001ED5, 0x0000006F}, {0x00001ED5, 0x00000302}, {0x00001ED5, 0x00000309}, -{0x00001ED6, 0x0000004F}, {0x00001ED6, 0x00000302}, {0x00001ED6, 0x00000303}, {0x00001ED7, 0x0000006F}, -{0x00001ED7, 0x00000302}, {0x00001ED7, 0x00000303}, {0x00001ED8, 0x0000004F}, {0x00001ED8, 0x00000323}, -{0x00001ED8, 0x00000302}, {0x00001ED9, 0x0000006F}, {0x00001ED9, 0x00000323}, {0x00001ED9, 0x00000302}, -{0x00001EDA, 0x0000004F}, {0x00001EDA, 0x0000031B}, {0x00001EDA, 0x00000301}, {0x00001EDB, 0x0000006F}, -{0x00001EDB, 0x0000031B}, {0x00001EDB, 0x00000301}, {0x00001EDC, 0x0000004F}, {0x00001EDC, 0x0000031B}, -{0x00001EDC, 0x00000300}, {0x00001EDD, 0x0000006F}, {0x00001EDD, 0x0000031B}, {0x00001EDD, 0x00000300}, -{0x00001EDE, 0x0000004F}, {0x00001EDE, 0x0000031B}, {0x00001EDE, 0x00000309}, {0x00001EDF, 0x0000006F}, -{0x00001EDF, 0x0000031B}, {0x00001EDF, 0x00000309}, {0x00001EE0, 0x0000004F}, {0x00001EE0, 0x0000031B}, -{0x00001EE0, 0x00000303}, {0x00001EE1, 0x0000006F}, {0x00001EE1, 0x0000031B}, {0x00001EE1, 0x00000303}, -{0x00001EE2, 0x0000004F}, {0x00001EE2, 0x0000031B}, {0x00001EE2, 0x00000323}, {0x00001EE3, 0x0000006F}, -{0x00001EE3, 0x0000031B}, {0x00001EE3, 0x00000323}, {0x00001EE4, 0x00000055}, {0x00001EE4, 0x00000323}, -{0x00001EE5, 0x00000075}, {0x00001EE5, 0x00000323}, {0x00001EE6, 0x00000055}, {0x00001EE6, 0x00000309}, -{0x00001EE7, 0x00000075}, {0x00001EE7, 0x00000309}, {0x00001EE8, 0x00000055}, {0x00001EE8, 0x0000031B}, -{0x00001EE8, 0x00000301}, {0x00001EE9, 0x00000075}, {0x00001EE9, 0x0000031B}, {0x00001EE9, 0x00000301}, -{0x00001EEA, 0x00000055}, {0x00001EEA, 0x0000031B}, {0x00001EEA, 0x00000300}, {0x00001EEB, 0x00000075}, -{0x00001EEB, 0x0000031B}, {0x00001EEB, 0x00000300}, {0x00001EEC, 0x00000055}, {0x00001EEC, 0x0000031B}, -{0x00001EEC, 0x00000309}, {0x00001EED, 0x00000075}, {0x00001EED, 0x0000031B}, {0x00001EED, 0x00000309}, -{0x00001EEE, 0x00000055}, {0x00001EEE, 0x0000031B}, {0x00001EEE, 0x00000303}, {0x00001EEF, 0x00000075}, -{0x00001EEF, 0x0000031B}, {0x00001EEF, 0x00000303}, {0x00001EF0, 0x00000055}, {0x00001EF0, 0x0000031B}, -{0x00001EF0, 0x00000323}, {0x00001EF1, 0x00000075}, {0x00001EF1, 0x0000031B}, {0x00001EF1, 0x00000323}, -{0x00001EF2, 0x00000059}, {0x00001EF2, 0x00000300}, {0x00001EF3, 0x00000079}, {0x00001EF3, 0x00000300}, -{0x00001EF4, 0x00000059}, {0x00001EF4, 0x00000323}, {0x00001EF5, 0x00000079}, {0x00001EF5, 0x00000323}, -{0x00001EF6, 0x00000059}, {0x00001EF6, 0x00000309}, {0x00001EF7, 0x00000079}, {0x00001EF7, 0x00000309}, -{0x00001EF8, 0x00000059}, {0x00001EF8, 0x00000303}, {0x00001EF9, 0x00000079}, {0x00001EF9, 0x00000303}, -{0x00001F00, 0x000003B1}, {0x00001F00, 0x00000313}, {0x00001F01, 0x000003B1}, {0x00001F01, 0x00000314}, -{0x00001F02, 0x000003B1}, {0x00001F02, 0x00000313}, {0x00001F02, 0x00000300}, {0x00001F03, 0x000003B1}, -{0x00001F03, 0x00000314}, {0x00001F03, 0x00000300}, {0x00001F04, 0x000003B1}, {0x00001F04, 0x00000313}, -{0x00001F04, 0x00000301}, {0x00001F05, 0x000003B1}, {0x00001F05, 0x00000314}, {0x00001F05, 0x00000301}, -{0x00001F06, 0x000003B1}, {0x00001F06, 0x00000313}, {0x00001F06, 0x00000342}, {0x00001F07, 0x000003B1}, -{0x00001F07, 0x00000314}, {0x00001F07, 0x00000342}, {0x00001F08, 0x00000391}, {0x00001F08, 0x00000313}, -{0x00001F09, 0x00000391}, {0x00001F09, 0x00000314}, {0x00001F0A, 0x00000391}, {0x00001F0A, 0x00000313}, -{0x00001F0A, 0x00000300}, {0x00001F0B, 0x00000391}, {0x00001F0B, 0x00000314}, {0x00001F0B, 0x00000300}, -{0x00001F0C, 0x00000391}, {0x00001F0C, 0x00000313}, {0x00001F0C, 0x00000301}, {0x00001F0D, 0x00000391}, -{0x00001F0D, 0x00000314}, {0x00001F0D, 0x00000301}, {0x00001F0E, 0x00000391}, {0x00001F0E, 0x00000313}, -{0x00001F0E, 0x00000342}, {0x00001F0F, 0x00000391}, {0x00001F0F, 0x00000314}, {0x00001F0F, 0x00000342}, -{0x00001F10, 0x000003B5}, {0x00001F10, 0x00000313}, {0x00001F11, 0x000003B5}, {0x00001F11, 0x00000314}, -{0x00001F12, 0x000003B5}, {0x00001F12, 0x00000313}, {0x00001F12, 0x00000300}, {0x00001F13, 0x000003B5}, -{0x00001F13, 0x00000314}, {0x00001F13, 0x00000300}, {0x00001F14, 0x000003B5}, {0x00001F14, 0x00000313}, -{0x00001F14, 0x00000301}, {0x00001F15, 0x000003B5}, {0x00001F15, 0x00000314}, {0x00001F15, 0x00000301}, -{0x00001F18, 0x00000395}, {0x00001F18, 0x00000313}, {0x00001F19, 0x00000395}, {0x00001F19, 0x00000314}, -{0x00001F1A, 0x00000395}, {0x00001F1A, 0x00000313}, {0x00001F1A, 0x00000300}, {0x00001F1B, 0x00000395}, -{0x00001F1B, 0x00000314}, {0x00001F1B, 0x00000300}, {0x00001F1C, 0x00000395}, {0x00001F1C, 0x00000313}, -{0x00001F1C, 0x00000301}, {0x00001F1D, 0x00000395}, {0x00001F1D, 0x00000314}, {0x00001F1D, 0x00000301}, -{0x00001F20, 0x000003B7}, {0x00001F20, 0x00000313}, {0x00001F21, 0x000003B7}, {0x00001F21, 0x00000314}, -{0x00001F22, 0x000003B7}, {0x00001F22, 0x00000313}, {0x00001F22, 0x00000300}, {0x00001F23, 0x000003B7}, -{0x00001F23, 0x00000314}, {0x00001F23, 0x00000300}, {0x00001F24, 0x000003B7}, {0x00001F24, 0x00000313}, -{0x00001F24, 0x00000301}, {0x00001F25, 0x000003B7}, {0x00001F25, 0x00000314}, {0x00001F25, 0x00000301}, -{0x00001F26, 0x000003B7}, {0x00001F26, 0x00000313}, {0x00001F26, 0x00000342}, {0x00001F27, 0x000003B7}, -{0x00001F27, 0x00000314}, {0x00001F27, 0x00000342}, {0x00001F28, 0x00000397}, {0x00001F28, 0x00000313}, -{0x00001F29, 0x00000397}, {0x00001F29, 0x00000314}, {0x00001F2A, 0x00000397}, {0x00001F2A, 0x00000313}, -{0x00001F2A, 0x00000300}, {0x00001F2B, 0x00000397}, {0x00001F2B, 0x00000314}, {0x00001F2B, 0x00000300}, -{0x00001F2C, 0x00000397}, {0x00001F2C, 0x00000313}, {0x00001F2C, 0x00000301}, {0x00001F2D, 0x00000397}, -{0x00001F2D, 0x00000314}, {0x00001F2D, 0x00000301}, {0x00001F2E, 0x00000397}, {0x00001F2E, 0x00000313}, -{0x00001F2E, 0x00000342}, {0x00001F2F, 0x00000397}, {0x00001F2F, 0x00000314}, {0x00001F2F, 0x00000342}, -{0x00001F30, 0x000003B9}, {0x00001F30, 0x00000313}, {0x00001F31, 0x000003B9}, {0x00001F31, 0x00000314}, -{0x00001F32, 0x000003B9}, {0x00001F32, 0x00000313}, {0x00001F32, 0x00000300}, {0x00001F33, 0x000003B9}, -{0x00001F33, 0x00000314}, {0x00001F33, 0x00000300}, {0x00001F34, 0x000003B9}, {0x00001F34, 0x00000313}, -{0x00001F34, 0x00000301}, {0x00001F35, 0x000003B9}, {0x00001F35, 0x00000314}, {0x00001F35, 0x00000301}, -{0x00001F36, 0x000003B9}, {0x00001F36, 0x00000313}, {0x00001F36, 0x00000342}, {0x00001F37, 0x000003B9}, -{0x00001F37, 0x00000314}, {0x00001F37, 0x00000342}, {0x00001F38, 0x00000399}, {0x00001F38, 0x00000313}, -{0x00001F39, 0x00000399}, {0x00001F39, 0x00000314}, {0x00001F3A, 0x00000399}, {0x00001F3A, 0x00000313}, -{0x00001F3A, 0x00000300}, {0x00001F3B, 0x00000399}, {0x00001F3B, 0x00000314}, {0x00001F3B, 0x00000300}, -{0x00001F3C, 0x00000399}, {0x00001F3C, 0x00000313}, {0x00001F3C, 0x00000301}, {0x00001F3D, 0x00000399}, -{0x00001F3D, 0x00000314}, {0x00001F3D, 0x00000301}, {0x00001F3E, 0x00000399}, {0x00001F3E, 0x00000313}, -{0x00001F3E, 0x00000342}, {0x00001F3F, 0x00000399}, {0x00001F3F, 0x00000314}, {0x00001F3F, 0x00000342}, -{0x00001F40, 0x000003BF}, {0x00001F40, 0x00000313}, {0x00001F41, 0x000003BF}, {0x00001F41, 0x00000314}, -{0x00001F42, 0x000003BF}, {0x00001F42, 0x00000313}, {0x00001F42, 0x00000300}, {0x00001F43, 0x000003BF}, -{0x00001F43, 0x00000314}, {0x00001F43, 0x00000300}, {0x00001F44, 0x000003BF}, {0x00001F44, 0x00000313}, -{0x00001F44, 0x00000301}, {0x00001F45, 0x000003BF}, {0x00001F45, 0x00000314}, {0x00001F45, 0x00000301}, -{0x00001F48, 0x0000039F}, {0x00001F48, 0x00000313}, {0x00001F49, 0x0000039F}, {0x00001F49, 0x00000314}, -{0x00001F4A, 0x0000039F}, {0x00001F4A, 0x00000313}, {0x00001F4A, 0x00000300}, {0x00001F4B, 0x0000039F}, -{0x00001F4B, 0x00000314}, {0x00001F4B, 0x00000300}, {0x00001F4C, 0x0000039F}, {0x00001F4C, 0x00000313}, -{0x00001F4C, 0x00000301}, {0x00001F4D, 0x0000039F}, {0x00001F4D, 0x00000314}, {0x00001F4D, 0x00000301}, -{0x00001F50, 0x000003C5}, {0x00001F50, 0x00000313}, {0x00001F51, 0x000003C5}, {0x00001F51, 0x00000314}, -{0x00001F52, 0x000003C5}, {0x00001F52, 0x00000313}, {0x00001F52, 0x00000300}, {0x00001F53, 0x000003C5}, -{0x00001F53, 0x00000314}, {0x00001F53, 0x00000300}, {0x00001F54, 0x000003C5}, {0x00001F54, 0x00000313}, -{0x00001F54, 0x00000301}, {0x00001F55, 0x000003C5}, {0x00001F55, 0x00000314}, {0x00001F55, 0x00000301}, -{0x00001F56, 0x000003C5}, {0x00001F56, 0x00000313}, {0x00001F56, 0x00000342}, {0x00001F57, 0x000003C5}, -{0x00001F57, 0x00000314}, {0x00001F57, 0x00000342}, {0x00001F59, 0x000003A5}, {0x00001F59, 0x00000314}, -{0x00001F5B, 0x000003A5}, {0x00001F5B, 0x00000314}, {0x00001F5B, 0x00000300}, {0x00001F5D, 0x000003A5}, -{0x00001F5D, 0x00000314}, {0x00001F5D, 0x00000301}, {0x00001F5F, 0x000003A5}, {0x00001F5F, 0x00000314}, -{0x00001F5F, 0x00000342}, {0x00001F60, 0x000003C9}, {0x00001F60, 0x00000313}, {0x00001F61, 0x000003C9}, -{0x00001F61, 0x00000314}, {0x00001F62, 0x000003C9}, {0x00001F62, 0x00000313}, {0x00001F62, 0x00000300}, -{0x00001F63, 0x000003C9}, {0x00001F63, 0x00000314}, {0x00001F63, 0x00000300}, {0x00001F64, 0x000003C9}, -{0x00001F64, 0x00000313}, {0x00001F64, 0x00000301}, {0x00001F65, 0x000003C9}, {0x00001F65, 0x00000314}, -{0x00001F65, 0x00000301}, {0x00001F66, 0x000003C9}, {0x00001F66, 0x00000313}, {0x00001F66, 0x00000342}, -{0x00001F67, 0x000003C9}, {0x00001F67, 0x00000314}, {0x00001F67, 0x00000342}, {0x00001F68, 0x000003A9}, -{0x00001F68, 0x00000313}, {0x00001F69, 0x000003A9}, {0x00001F69, 0x00000314}, {0x00001F6A, 0x000003A9}, -{0x00001F6A, 0x00000313}, {0x00001F6A, 0x00000300}, {0x00001F6B, 0x000003A9}, {0x00001F6B, 0x00000314}, -{0x00001F6B, 0x00000300}, {0x00001F6C, 0x000003A9}, {0x00001F6C, 0x00000313}, {0x00001F6C, 0x00000301}, -{0x00001F6D, 0x000003A9}, {0x00001F6D, 0x00000314}, {0x00001F6D, 0x00000301}, {0x00001F6E, 0x000003A9}, -{0x00001F6E, 0x00000313}, {0x00001F6E, 0x00000342}, {0x00001F6F, 0x000003A9}, {0x00001F6F, 0x00000314}, -{0x00001F6F, 0x00000342}, {0x00001F70, 0x000003B1}, {0x00001F70, 0x00000300}, {0x00001F71, 0x000003B1}, -{0x00001F71, 0x00000301}, {0x00001F72, 0x000003B5}, {0x00001F72, 0x00000300}, {0x00001F73, 0x000003B5}, -{0x00001F73, 0x00000301}, {0x00001F74, 0x000003B7}, {0x00001F74, 0x00000300}, {0x00001F75, 0x000003B7}, -{0x00001F75, 0x00000301}, {0x00001F76, 0x000003B9}, {0x00001F76, 0x00000300}, {0x00001F77, 0x000003B9}, -{0x00001F77, 0x00000301}, {0x00001F78, 0x000003BF}, {0x00001F78, 0x00000300}, {0x00001F79, 0x000003BF}, -{0x00001F79, 0x00000301}, {0x00001F7A, 0x000003C5}, {0x00001F7A, 0x00000300}, {0x00001F7B, 0x000003C5}, -{0x00001F7B, 0x00000301}, {0x00001F7C, 0x000003C9}, {0x00001F7C, 0x00000300}, {0x00001F7D, 0x000003C9}, -{0x00001F7D, 0x00000301}, {0x00001F80, 0x000003B1}, {0x00001F80, 0x00000313}, {0x00001F80, 0x00000345}, -{0x00001F81, 0x000003B1}, {0x00001F81, 0x00000314}, {0x00001F81, 0x00000345}, {0x00001F82, 0x000003B1}, -{0x00001F82, 0x00000313}, {0x00001F82, 0x00000300}, {0x00001F82, 0x00000345}, {0x00001F83, 0x000003B1}, -{0x00001F83, 0x00000314}, {0x00001F83, 0x00000300}, {0x00001F83, 0x00000345}, {0x00001F84, 0x000003B1}, -{0x00001F84, 0x00000313}, {0x00001F84, 0x00000301}, {0x00001F84, 0x00000345}, {0x00001F85, 0x000003B1}, -{0x00001F85, 0x00000314}, {0x00001F85, 0x00000301}, {0x00001F85, 0x00000345}, {0x00001F86, 0x000003B1}, -{0x00001F86, 0x00000313}, {0x00001F86, 0x00000342}, {0x00001F86, 0x00000345}, {0x00001F87, 0x000003B1}, -{0x00001F87, 0x00000314}, {0x00001F87, 0x00000342}, {0x00001F87, 0x00000345}, {0x00001F88, 0x00000391}, -{0x00001F88, 0x00000313}, {0x00001F88, 0x00000345}, {0x00001F89, 0x00000391}, {0x00001F89, 0x00000314}, -{0x00001F89, 0x00000345}, {0x00001F8A, 0x00000391}, {0x00001F8A, 0x00000313}, {0x00001F8A, 0x00000300}, -{0x00001F8A, 0x00000345}, {0x00001F8B, 0x00000391}, {0x00001F8B, 0x00000314}, {0x00001F8B, 0x00000300}, -{0x00001F8B, 0x00000345}, {0x00001F8C, 0x00000391}, {0x00001F8C, 0x00000313}, {0x00001F8C, 0x00000301}, -{0x00001F8C, 0x00000345}, {0x00001F8D, 0x00000391}, {0x00001F8D, 0x00000314}, {0x00001F8D, 0x00000301}, -{0x00001F8D, 0x00000345}, {0x00001F8E, 0x00000391}, {0x00001F8E, 0x00000313}, {0x00001F8E, 0x00000342}, -{0x00001F8E, 0x00000345}, {0x00001F8F, 0x00000391}, {0x00001F8F, 0x00000314}, {0x00001F8F, 0x00000342}, -{0x00001F8F, 0x00000345}, {0x00001F90, 0x000003B7}, {0x00001F90, 0x00000313}, {0x00001F90, 0x00000345}, -{0x00001F91, 0x000003B7}, {0x00001F91, 0x00000314}, {0x00001F91, 0x00000345}, {0x00001F92, 0x000003B7}, -{0x00001F92, 0x00000313}, {0x00001F92, 0x00000300}, {0x00001F92, 0x00000345}, {0x00001F93, 0x000003B7}, -{0x00001F93, 0x00000314}, {0x00001F93, 0x00000300}, {0x00001F93, 0x00000345}, {0x00001F94, 0x000003B7}, -{0x00001F94, 0x00000313}, {0x00001F94, 0x00000301}, {0x00001F94, 0x00000345}, {0x00001F95, 0x000003B7}, -{0x00001F95, 0x00000314}, {0x00001F95, 0x00000301}, {0x00001F95, 0x00000345}, {0x00001F96, 0x000003B7}, -{0x00001F96, 0x00000313}, {0x00001F96, 0x00000342}, {0x00001F96, 0x00000345}, {0x00001F97, 0x000003B7}, -{0x00001F97, 0x00000314}, {0x00001F97, 0x00000342}, {0x00001F97, 0x00000345}, {0x00001F98, 0x00000397}, -{0x00001F98, 0x00000313}, {0x00001F98, 0x00000345}, {0x00001F99, 0x00000397}, {0x00001F99, 0x00000314}, -{0x00001F99, 0x00000345}, {0x00001F9A, 0x00000397}, {0x00001F9A, 0x00000313}, {0x00001F9A, 0x00000300}, -{0x00001F9A, 0x00000345}, {0x00001F9B, 0x00000397}, {0x00001F9B, 0x00000314}, {0x00001F9B, 0x00000300}, -{0x00001F9B, 0x00000345}, {0x00001F9C, 0x00000397}, {0x00001F9C, 0x00000313}, {0x00001F9C, 0x00000301}, -{0x00001F9C, 0x00000345}, {0x00001F9D, 0x00000397}, {0x00001F9D, 0x00000314}, {0x00001F9D, 0x00000301}, -{0x00001F9D, 0x00000345}, {0x00001F9E, 0x00000397}, {0x00001F9E, 0x00000313}, {0x00001F9E, 0x00000342}, -{0x00001F9E, 0x00000345}, {0x00001F9F, 0x00000397}, {0x00001F9F, 0x00000314}, {0x00001F9F, 0x00000342}, -{0x00001F9F, 0x00000345}, {0x00001FA0, 0x000003C9}, {0x00001FA0, 0x00000313}, {0x00001FA0, 0x00000345}, -{0x00001FA1, 0x000003C9}, {0x00001FA1, 0x00000314}, {0x00001FA1, 0x00000345}, {0x00001FA2, 0x000003C9}, -{0x00001FA2, 0x00000313}, {0x00001FA2, 0x00000300}, {0x00001FA2, 0x00000345}, {0x00001FA3, 0x000003C9}, -{0x00001FA3, 0x00000314}, {0x00001FA3, 0x00000300}, {0x00001FA3, 0x00000345}, {0x00001FA4, 0x000003C9}, -{0x00001FA4, 0x00000313}, {0x00001FA4, 0x00000301}, {0x00001FA4, 0x00000345}, {0x00001FA5, 0x000003C9}, -{0x00001FA5, 0x00000314}, {0x00001FA5, 0x00000301}, {0x00001FA5, 0x00000345}, {0x00001FA6, 0x000003C9}, -{0x00001FA6, 0x00000313}, {0x00001FA6, 0x00000342}, {0x00001FA6, 0x00000345}, {0x00001FA7, 0x000003C9}, -{0x00001FA7, 0x00000314}, {0x00001FA7, 0x00000342}, {0x00001FA7, 0x00000345}, {0x00001FA8, 0x000003A9}, -{0x00001FA8, 0x00000313}, {0x00001FA8, 0x00000345}, {0x00001FA9, 0x000003A9}, {0x00001FA9, 0x00000314}, -{0x00001FA9, 0x00000345}, {0x00001FAA, 0x000003A9}, {0x00001FAA, 0x00000313}, {0x00001FAA, 0x00000300}, -{0x00001FAA, 0x00000345}, {0x00001FAB, 0x000003A9}, {0x00001FAB, 0x00000314}, {0x00001FAB, 0x00000300}, -{0x00001FAB, 0x00000345}, {0x00001FAC, 0x000003A9}, {0x00001FAC, 0x00000313}, {0x00001FAC, 0x00000301}, -{0x00001FAC, 0x00000345}, {0x00001FAD, 0x000003A9}, {0x00001FAD, 0x00000314}, {0x00001FAD, 0x00000301}, -{0x00001FAD, 0x00000345}, {0x00001FAE, 0x000003A9}, {0x00001FAE, 0x00000313}, {0x00001FAE, 0x00000342}, -{0x00001FAE, 0x00000345}, {0x00001FAF, 0x000003A9}, {0x00001FAF, 0x00000314}, {0x00001FAF, 0x00000342}, -{0x00001FAF, 0x00000345}, {0x00001FB0, 0x000003B1}, {0x00001FB0, 0x00000306}, {0x00001FB1, 0x000003B1}, -{0x00001FB1, 0x00000304}, {0x00001FB2, 0x000003B1}, {0x00001FB2, 0x00000300}, {0x00001FB2, 0x00000345}, -{0x00001FB3, 0x000003B1}, {0x00001FB3, 0x00000345}, {0x00001FB4, 0x000003B1}, {0x00001FB4, 0x00000301}, -{0x00001FB4, 0x00000345}, {0x00001FB6, 0x000003B1}, {0x00001FB6, 0x00000342}, {0x00001FB7, 0x000003B1}, -{0x00001FB7, 0x00000342}, {0x00001FB7, 0x00000345}, {0x00001FB8, 0x00000391}, {0x00001FB8, 0x00000306}, -{0x00001FB9, 0x00000391}, {0x00001FB9, 0x00000304}, {0x00001FBA, 0x00000391}, {0x00001FBA, 0x00000300}, -{0x00001FBB, 0x00000391}, {0x00001FBB, 0x00000301}, {0x00001FBC, 0x00000391}, {0x00001FBC, 0x00000345}, -{0x00001FBE, 0x000003B9}, {0x00001FC1, 0x000000A8}, {0x00001FC1, 0x00000342}, {0x00001FC2, 0x000003B7}, -{0x00001FC2, 0x00000300}, {0x00001FC2, 0x00000345}, {0x00001FC3, 0x000003B7}, {0x00001FC3, 0x00000345}, -{0x00001FC4, 0x000003B7}, {0x00001FC4, 0x00000301}, {0x00001FC4, 0x00000345}, {0x00001FC6, 0x000003B7}, -{0x00001FC6, 0x00000342}, {0x00001FC7, 0x000003B7}, {0x00001FC7, 0x00000342}, {0x00001FC7, 0x00000345}, -{0x00001FC8, 0x00000395}, {0x00001FC8, 0x00000300}, {0x00001FC9, 0x00000395}, {0x00001FC9, 0x00000301}, -{0x00001FCA, 0x00000397}, {0x00001FCA, 0x00000300}, {0x00001FCB, 0x00000397}, {0x00001FCB, 0x00000301}, -{0x00001FCC, 0x00000397}, {0x00001FCC, 0x00000345}, {0x00001FCD, 0x00001FBF}, {0x00001FCD, 0x00000300}, -{0x00001FCE, 0x00001FBF}, {0x00001FCE, 0x00000301}, {0x00001FCF, 0x00001FBF}, {0x00001FCF, 0x00000342}, -{0x00001FD0, 0x000003B9}, {0x00001FD0, 0x00000306}, {0x00001FD1, 0x000003B9}, {0x00001FD1, 0x00000304}, -{0x00001FD2, 0x000003B9}, {0x00001FD2, 0x00000308}, {0x00001FD2, 0x00000300}, {0x00001FD3, 0x000003B9}, -{0x00001FD3, 0x00000308}, {0x00001FD3, 0x00000301}, {0x00001FD6, 0x000003B9}, {0x00001FD6, 0x00000342}, -{0x00001FD7, 0x000003B9}, {0x00001FD7, 0x00000308}, {0x00001FD7, 0x00000342}, {0x00001FD8, 0x00000399}, -{0x00001FD8, 0x00000306}, {0x00001FD9, 0x00000399}, {0x00001FD9, 0x00000304}, {0x00001FDA, 0x00000399}, -{0x00001FDA, 0x00000300}, {0x00001FDB, 0x00000399}, {0x00001FDB, 0x00000301}, {0x00001FDD, 0x00001FFE}, -{0x00001FDD, 0x00000300}, {0x00001FDE, 0x00001FFE}, {0x00001FDE, 0x00000301}, {0x00001FDF, 0x00001FFE}, -{0x00001FDF, 0x00000342}, {0x00001FE0, 0x000003C5}, {0x00001FE0, 0x00000306}, {0x00001FE1, 0x000003C5}, -{0x00001FE1, 0x00000304}, {0x00001FE2, 0x000003C5}, {0x00001FE2, 0x00000308}, {0x00001FE2, 0x00000300}, -{0x00001FE3, 0x000003C5}, {0x00001FE3, 0x00000308}, {0x00001FE3, 0x00000301}, {0x00001FE4, 0x000003C1}, -{0x00001FE4, 0x00000313}, {0x00001FE5, 0x000003C1}, {0x00001FE5, 0x00000314}, {0x00001FE6, 0x000003C5}, -{0x00001FE6, 0x00000342}, {0x00001FE7, 0x000003C5}, {0x00001FE7, 0x00000308}, {0x00001FE7, 0x00000342}, -{0x00001FE8, 0x000003A5}, {0x00001FE8, 0x00000306}, {0x00001FE9, 0x000003A5}, {0x00001FE9, 0x00000304}, -{0x00001FEA, 0x000003A5}, {0x00001FEA, 0x00000300}, {0x00001FEB, 0x000003A5}, {0x00001FEB, 0x00000301}, -{0x00001FEC, 0x000003A1}, {0x00001FEC, 0x00000314}, {0x00001FED, 0x000000A8}, {0x00001FED, 0x00000300}, -{0x00001FEE, 0x000000A8}, {0x00001FEE, 0x00000301}, {0x00001FEF, 0x00000060}, {0x00001FF2, 0x000003C9}, -{0x00001FF2, 0x00000300}, {0x00001FF2, 0x00000345}, {0x00001FF3, 0x000003C9}, {0x00001FF3, 0x00000345}, -{0x00001FF4, 0x000003C9}, {0x00001FF4, 0x00000301}, {0x00001FF4, 0x00000345}, {0x00001FF6, 0x000003C9}, -{0x00001FF6, 0x00000342}, {0x00001FF7, 0x000003C9}, {0x00001FF7, 0x00000342}, {0x00001FF7, 0x00000345}, -{0x00001FF8, 0x0000039F}, {0x00001FF8, 0x00000300}, {0x00001FF9, 0x0000039F}, {0x00001FF9, 0x00000301}, -{0x00001FFA, 0x000003A9}, {0x00001FFA, 0x00000300}, {0x00001FFB, 0x000003A9}, {0x00001FFB, 0x00000301}, -{0x00001FFC, 0x000003A9}, {0x00001FFC, 0x00000345}, {0x00001FFD, 0x000000B4}, {0x00002000, 0x00002002}, -{0x00002001, 0x00002003}, {0x00002126, 0x000003A9}, {0x0000212A, 0x0000004B}, {0x0000212B, 0x00000041}, -{0x0000212B, 0x0000030A}, {0x0000219A, 0x00002190}, {0x0000219A, 0x00000338}, {0x0000219B, 0x00002192}, -{0x0000219B, 0x00000338}, {0x000021AE, 0x00002194}, {0x000021AE, 0x00000338}, {0x000021CD, 0x000021D0}, -{0x000021CD, 0x00000338}, {0x000021CE, 0x000021D4}, {0x000021CE, 0x00000338}, {0x000021CF, 0x000021D2}, -{0x000021CF, 0x00000338}, {0x00002204, 0x00002203}, {0x00002204, 0x00000338}, {0x00002209, 0x00002208}, -{0x00002209, 0x00000338}, {0x0000220C, 0x0000220B}, {0x0000220C, 0x00000338}, {0x00002224, 0x00002223}, -{0x00002224, 0x00000338}, {0x00002226, 0x00002225}, {0x00002226, 0x00000338}, {0x00002241, 0x0000223C}, -{0x00002241, 0x00000338}, {0x00002244, 0x00002243}, {0x00002244, 0x00000338}, {0x00002247, 0x00002245}, -{0x00002247, 0x00000338}, {0x00002249, 0x00002248}, {0x00002249, 0x00000338}, {0x00002260, 0x0000003D}, -{0x00002260, 0x00000338}, {0x00002262, 0x00002261}, {0x00002262, 0x00000338}, {0x0000226D, 0x0000224D}, -{0x0000226D, 0x00000338}, {0x0000226E, 0x0000003C}, {0x0000226E, 0x00000338}, {0x0000226F, 0x0000003E}, -{0x0000226F, 0x00000338}, {0x00002270, 0x00002264}, {0x00002270, 0x00000338}, {0x00002271, 0x00002265}, -{0x00002271, 0x00000338}, {0x00002274, 0x00002272}, {0x00002274, 0x00000338}, {0x00002275, 0x00002273}, -{0x00002275, 0x00000338}, {0x00002278, 0x00002276}, {0x00002278, 0x00000338}, {0x00002279, 0x00002277}, -{0x00002279, 0x00000338}, {0x00002280, 0x0000227A}, {0x00002280, 0x00000338}, {0x00002281, 0x0000227B}, -{0x00002281, 0x00000338}, {0x00002284, 0x00002282}, {0x00002284, 0x00000338}, {0x00002285, 0x00002283}, -{0x00002285, 0x00000338}, {0x00002288, 0x00002286}, {0x00002288, 0x00000338}, {0x00002289, 0x00002287}, -{0x00002289, 0x00000338}, {0x000022AC, 0x000022A2}, {0x000022AC, 0x00000338}, {0x000022AD, 0x000022A8}, -{0x000022AD, 0x00000338}, {0x000022AE, 0x000022A9}, {0x000022AE, 0x00000338}, {0x000022AF, 0x000022AB}, -{0x000022AF, 0x00000338}, {0x000022E0, 0x0000227C}, {0x000022E0, 0x00000338}, {0x000022E1, 0x0000227D}, -{0x000022E1, 0x00000338}, {0x000022E2, 0x00002291}, {0x000022E2, 0x00000338}, {0x000022E3, 0x00002292}, -{0x000022E3, 0x00000338}, {0x000022EA, 0x000022B2}, {0x000022EA, 0x00000338}, {0x000022EB, 0x000022B3}, -{0x000022EB, 0x00000338}, {0x000022EC, 0x000022B4}, {0x000022EC, 0x00000338}, {0x000022ED, 0x000022B5}, -{0x000022ED, 0x00000338}, {0x00002329, 0x00003008}, {0x0000232A, 0x00003009}, {0x00002ADC, 0x00002ADD}, -{0x00002ADC, 0x00000338}, {0x0000304C, 0x0000304B}, {0x0000304C, 0x00003099}, {0x0000304E, 0x0000304D}, -{0x0000304E, 0x00003099}, {0x00003050, 0x0000304F}, {0x00003050, 0x00003099}, {0x00003052, 0x00003051}, -{0x00003052, 0x00003099}, {0x00003054, 0x00003053}, {0x00003054, 0x00003099}, {0x00003056, 0x00003055}, -{0x00003056, 0x00003099}, {0x00003058, 0x00003057}, {0x00003058, 0x00003099}, {0x0000305A, 0x00003059}, -{0x0000305A, 0x00003099}, {0x0000305C, 0x0000305B}, {0x0000305C, 0x00003099}, {0x0000305E, 0x0000305D}, -{0x0000305E, 0x00003099}, {0x00003060, 0x0000305F}, {0x00003060, 0x00003099}, {0x00003062, 0x00003061}, -{0x00003062, 0x00003099}, {0x00003065, 0x00003064}, {0x00003065, 0x00003099}, {0x00003067, 0x00003066}, -{0x00003067, 0x00003099}, {0x00003069, 0x00003068}, {0x00003069, 0x00003099}, {0x00003070, 0x0000306F}, -{0x00003070, 0x00003099}, {0x00003071, 0x0000306F}, {0x00003071, 0x0000309A}, {0x00003073, 0x00003072}, -{0x00003073, 0x00003099}, {0x00003074, 0x00003072}, {0x00003074, 0x0000309A}, {0x00003076, 0x00003075}, -{0x00003076, 0x00003099}, {0x00003077, 0x00003075}, {0x00003077, 0x0000309A}, {0x00003079, 0x00003078}, -{0x00003079, 0x00003099}, {0x0000307A, 0x00003078}, {0x0000307A, 0x0000309A}, {0x0000307C, 0x0000307B}, -{0x0000307C, 0x00003099}, {0x0000307D, 0x0000307B}, {0x0000307D, 0x0000309A}, {0x00003094, 0x00003046}, -{0x00003094, 0x00003099}, {0x0000309E, 0x0000309D}, {0x0000309E, 0x00003099}, {0x000030AC, 0x000030AB}, -{0x000030AC, 0x00003099}, {0x000030AE, 0x000030AD}, {0x000030AE, 0x00003099}, {0x000030B0, 0x000030AF}, -{0x000030B0, 0x00003099}, {0x000030B2, 0x000030B1}, {0x000030B2, 0x00003099}, {0x000030B4, 0x000030B3}, -{0x000030B4, 0x00003099}, {0x000030B6, 0x000030B5}, {0x000030B6, 0x00003099}, {0x000030B8, 0x000030B7}, -{0x000030B8, 0x00003099}, {0x000030BA, 0x000030B9}, {0x000030BA, 0x00003099}, {0x000030BC, 0x000030BB}, -{0x000030BC, 0x00003099}, {0x000030BE, 0x000030BD}, {0x000030BE, 0x00003099}, {0x000030C0, 0x000030BF}, -{0x000030C0, 0x00003099}, {0x000030C2, 0x000030C1}, {0x000030C2, 0x00003099}, {0x000030C5, 0x000030C4}, -{0x000030C5, 0x00003099}, {0x000030C7, 0x000030C6}, {0x000030C7, 0x00003099}, {0x000030C9, 0x000030C8}, -{0x000030C9, 0x00003099}, {0x000030D0, 0x000030CF}, {0x000030D0, 0x00003099}, {0x000030D1, 0x000030CF}, -{0x000030D1, 0x0000309A}, {0x000030D3, 0x000030D2}, {0x000030D3, 0x00003099}, {0x000030D4, 0x000030D2}, -{0x000030D4, 0x0000309A}, {0x000030D6, 0x000030D5}, {0x000030D6, 0x00003099}, {0x000030D7, 0x000030D5}, -{0x000030D7, 0x0000309A}, {0x000030D9, 0x000030D8}, {0x000030D9, 0x00003099}, {0x000030DA, 0x000030D8}, -{0x000030DA, 0x0000309A}, {0x000030DC, 0x000030DB}, {0x000030DC, 0x00003099}, {0x000030DD, 0x000030DB}, -{0x000030DD, 0x0000309A}, {0x000030F4, 0x000030A6}, {0x000030F4, 0x00003099}, {0x000030F7, 0x000030EF}, -{0x000030F7, 0x00003099}, {0x000030F8, 0x000030F0}, {0x000030F8, 0x00003099}, {0x000030F9, 0x000030F1}, -{0x000030F9, 0x00003099}, {0x000030FA, 0x000030F2}, {0x000030FA, 0x00003099}, {0x000030FE, 0x000030FD}, -{0x000030FE, 0x00003099}, {0x0000F900, 0x00008C48}, {0x0000F901, 0x000066F4}, {0x0000F902, 0x00008ECA}, -{0x0000F903, 0x00008CC8}, {0x0000F904, 0x00006ED1}, {0x0000F905, 0x00004E32}, {0x0000F906, 0x000053E5}, -{0x0000F907, 0x00009F9C}, {0x0000F908, 0x00009F9C}, {0x0000F909, 0x00005951}, {0x0000F90A, 0x000091D1}, -{0x0000F90B, 0x00005587}, {0x0000F90C, 0x00005948}, {0x0000F90D, 0x000061F6}, {0x0000F90E, 0x00007669}, -{0x0000F90F, 0x00007F85}, {0x0000F910, 0x0000863F}, {0x0000F911, 0x000087BA}, {0x0000F912, 0x000088F8}, -{0x0000F913, 0x0000908F}, {0x0000F914, 0x00006A02}, {0x0000F915, 0x00006D1B}, {0x0000F916, 0x000070D9}, -{0x0000F917, 0x000073DE}, {0x0000F918, 0x0000843D}, {0x0000F919, 0x0000916A}, {0x0000F91A, 0x000099F1}, -{0x0000F91B, 0x00004E82}, {0x0000F91C, 0x00005375}, {0x0000F91D, 0x00006B04}, {0x0000F91E, 0x0000721B}, -{0x0000F91F, 0x0000862D}, {0x0000F920, 0x00009E1E}, {0x0000F921, 0x00005D50}, {0x0000F922, 0x00006FEB}, -{0x0000F923, 0x000085CD}, {0x0000F924, 0x00008964}, {0x0000F925, 0x000062C9}, {0x0000F926, 0x000081D8}, -{0x0000F927, 0x0000881F}, {0x0000F928, 0x00005ECA}, {0x0000F929, 0x00006717}, {0x0000F92A, 0x00006D6A}, -{0x0000F92B, 0x000072FC}, {0x0000F92C, 0x000090CE}, {0x0000F92D, 0x00004F86}, {0x0000F92E, 0x000051B7}, -{0x0000F92F, 0x000052DE}, {0x0000F930, 0x000064C4}, {0x0000F931, 0x00006AD3}, {0x0000F932, 0x00007210}, -{0x0000F933, 0x000076E7}, {0x0000F934, 0x00008001}, {0x0000F935, 0x00008606}, {0x0000F936, 0x0000865C}, -{0x0000F937, 0x00008DEF}, {0x0000F938, 0x00009732}, {0x0000F939, 0x00009B6F}, {0x0000F93A, 0x00009DFA}, -{0x0000F93B, 0x0000788C}, {0x0000F93C, 0x0000797F}, {0x0000F93D, 0x00007DA0}, {0x0000F93E, 0x000083C9}, -{0x0000F93F, 0x00009304}, {0x0000F940, 0x00009E7F}, {0x0000F941, 0x00008AD6}, {0x0000F942, 0x000058DF}, -{0x0000F943, 0x00005F04}, {0x0000F944, 0x00007C60}, {0x0000F945, 0x0000807E}, {0x0000F946, 0x00007262}, -{0x0000F947, 0x000078CA}, {0x0000F948, 0x00008CC2}, {0x0000F949, 0x000096F7}, {0x0000F94A, 0x000058D8}, -{0x0000F94B, 0x00005C62}, {0x0000F94C, 0x00006A13}, {0x0000F94D, 0x00006DDA}, {0x0000F94E, 0x00006F0F}, -{0x0000F94F, 0x00007D2F}, {0x0000F950, 0x00007E37}, {0x0000F951, 0x0000964B}, {0x0000F952, 0x000052D2}, -{0x0000F953, 0x0000808B}, {0x0000F954, 0x000051DC}, {0x0000F955, 0x000051CC}, {0x0000F956, 0x00007A1C}, -{0x0000F957, 0x00007DBE}, {0x0000F958, 0x000083F1}, {0x0000F959, 0x00009675}, {0x0000F95A, 0x00008B80}, -{0x0000F95B, 0x000062CF}, {0x0000F95C, 0x00006A02}, {0x0000F95D, 0x00008AFE}, {0x0000F95E, 0x00004E39}, -{0x0000F95F, 0x00005BE7}, {0x0000F960, 0x00006012}, {0x0000F961, 0x00007387}, {0x0000F962, 0x00007570}, -{0x0000F963, 0x00005317}, {0x0000F964, 0x000078FB}, {0x0000F965, 0x00004FBF}, {0x0000F966, 0x00005FA9}, -{0x0000F967, 0x00004E0D}, {0x0000F968, 0x00006CCC}, {0x0000F969, 0x00006578}, {0x0000F96A, 0x00007D22}, -{0x0000F96B, 0x000053C3}, {0x0000F96C, 0x0000585E}, {0x0000F96D, 0x00007701}, {0x0000F96E, 0x00008449}, -{0x0000F96F, 0x00008AAA}, {0x0000F970, 0x00006BBA}, {0x0000F971, 0x00008FB0}, {0x0000F972, 0x00006C88}, -{0x0000F973, 0x000062FE}, {0x0000F974, 0x000082E5}, {0x0000F975, 0x000063A0}, {0x0000F976, 0x00007565}, -{0x0000F977, 0x00004EAE}, {0x0000F978, 0x00005169}, {0x0000F979, 0x000051C9}, {0x0000F97A, 0x00006881}, -{0x0000F97B, 0x00007CE7}, {0x0000F97C, 0x0000826F}, {0x0000F97D, 0x00008AD2}, {0x0000F97E, 0x000091CF}, -{0x0000F97F, 0x000052F5}, {0x0000F980, 0x00005442}, {0x0000F981, 0x00005973}, {0x0000F982, 0x00005EEC}, -{0x0000F983, 0x000065C5}, {0x0000F984, 0x00006FFE}, {0x0000F985, 0x0000792A}, {0x0000F986, 0x000095AD}, -{0x0000F987, 0x00009A6A}, {0x0000F988, 0x00009E97}, {0x0000F989, 0x00009ECE}, {0x0000F98A, 0x0000529B}, -{0x0000F98B, 0x000066C6}, {0x0000F98C, 0x00006B77}, {0x0000F98D, 0x00008F62}, {0x0000F98E, 0x00005E74}, -{0x0000F98F, 0x00006190}, {0x0000F990, 0x00006200}, {0x0000F991, 0x0000649A}, {0x0000F992, 0x00006F23}, -{0x0000F993, 0x00007149}, {0x0000F994, 0x00007489}, {0x0000F995, 0x000079CA}, {0x0000F996, 0x00007DF4}, -{0x0000F997, 0x0000806F}, {0x0000F998, 0x00008F26}, {0x0000F999, 0x000084EE}, {0x0000F99A, 0x00009023}, -{0x0000F99B, 0x0000934A}, {0x0000F99C, 0x00005217}, {0x0000F99D, 0x000052A3}, {0x0000F99E, 0x000054BD}, -{0x0000F99F, 0x000070C8}, {0x0000F9A0, 0x000088C2}, {0x0000F9A1, 0x00008AAA}, {0x0000F9A2, 0x00005EC9}, -{0x0000F9A3, 0x00005FF5}, {0x0000F9A4, 0x0000637B}, {0x0000F9A5, 0x00006BAE}, {0x0000F9A6, 0x00007C3E}, -{0x0000F9A7, 0x00007375}, {0x0000F9A8, 0x00004EE4}, {0x0000F9A9, 0x000056F9}, {0x0000F9AA, 0x00005BE7}, -{0x0000F9AB, 0x00005DBA}, {0x0000F9AC, 0x0000601C}, {0x0000F9AD, 0x000073B2}, {0x0000F9AE, 0x00007469}, -{0x0000F9AF, 0x00007F9A}, {0x0000F9B0, 0x00008046}, {0x0000F9B1, 0x00009234}, {0x0000F9B2, 0x000096F6}, -{0x0000F9B3, 0x00009748}, {0x0000F9B4, 0x00009818}, {0x0000F9B5, 0x00004F8B}, {0x0000F9B6, 0x000079AE}, -{0x0000F9B7, 0x000091B4}, {0x0000F9B8, 0x000096B8}, {0x0000F9B9, 0x000060E1}, {0x0000F9BA, 0x00004E86}, -{0x0000F9BB, 0x000050DA}, {0x0000F9BC, 0x00005BEE}, {0x0000F9BD, 0x00005C3F}, {0x0000F9BE, 0x00006599}, -{0x0000F9BF, 0x00006A02}, {0x0000F9C0, 0x000071CE}, {0x0000F9C1, 0x00007642}, {0x0000F9C2, 0x000084FC}, -{0x0000F9C3, 0x0000907C}, {0x0000F9C4, 0x00009F8D}, {0x0000F9C5, 0x00006688}, {0x0000F9C6, 0x0000962E}, -{0x0000F9C7, 0x00005289}, {0x0000F9C8, 0x0000677B}, {0x0000F9C9, 0x000067F3}, {0x0000F9CA, 0x00006D41}, -{0x0000F9CB, 0x00006E9C}, {0x0000F9CC, 0x00007409}, {0x0000F9CD, 0x00007559}, {0x0000F9CE, 0x0000786B}, -{0x0000F9CF, 0x00007D10}, {0x0000F9D0, 0x0000985E}, {0x0000F9D1, 0x0000516D}, {0x0000F9D2, 0x0000622E}, -{0x0000F9D3, 0x00009678}, {0x0000F9D4, 0x0000502B}, {0x0000F9D5, 0x00005D19}, {0x0000F9D6, 0x00006DEA}, -{0x0000F9D7, 0x00008F2A}, {0x0000F9D8, 0x00005F8B}, {0x0000F9D9, 0x00006144}, {0x0000F9DA, 0x00006817}, -{0x0000F9DB, 0x00007387}, {0x0000F9DC, 0x00009686}, {0x0000F9DD, 0x00005229}, {0x0000F9DE, 0x0000540F}, -{0x0000F9DF, 0x00005C65}, {0x0000F9E0, 0x00006613}, {0x0000F9E1, 0x0000674E}, {0x0000F9E2, 0x000068A8}, -{0x0000F9E3, 0x00006CE5}, {0x0000F9E4, 0x00007406}, {0x0000F9E5, 0x000075E2}, {0x0000F9E6, 0x00007F79}, -{0x0000F9E7, 0x000088CF}, {0x0000F9E8, 0x000088E1}, {0x0000F9E9, 0x000091CC}, {0x0000F9EA, 0x000096E2}, -{0x0000F9EB, 0x0000533F}, {0x0000F9EC, 0x00006EBA}, {0x0000F9ED, 0x0000541D}, {0x0000F9EE, 0x000071D0}, -{0x0000F9EF, 0x00007498}, {0x0000F9F0, 0x000085FA}, {0x0000F9F1, 0x000096A3}, {0x0000F9F2, 0x00009C57}, -{0x0000F9F3, 0x00009E9F}, {0x0000F9F4, 0x00006797}, {0x0000F9F5, 0x00006DCB}, {0x0000F9F6, 0x000081E8}, -{0x0000F9F7, 0x00007ACB}, {0x0000F9F8, 0x00007B20}, {0x0000F9F9, 0x00007C92}, {0x0000F9FA, 0x000072C0}, -{0x0000F9FB, 0x00007099}, {0x0000F9FC, 0x00008B58}, {0x0000F9FD, 0x00004EC0}, {0x0000F9FE, 0x00008336}, -{0x0000F9FF, 0x0000523A}, {0x0000FA00, 0x00005207}, {0x0000FA01, 0x00005EA6}, {0x0000FA02, 0x000062D3}, -{0x0000FA03, 0x00007CD6}, {0x0000FA04, 0x00005B85}, {0x0000FA05, 0x00006D1E}, {0x0000FA06, 0x000066B4}, -{0x0000FA07, 0x00008F3B}, {0x0000FA08, 0x0000884C}, {0x0000FA09, 0x0000964D}, {0x0000FA0A, 0x0000898B}, -{0x0000FA0B, 0x00005ED3}, {0x0000FA0C, 0x00005140}, {0x0000FA0D, 0x000055C0}, {0x0000FA10, 0x0000585A}, -{0x0000FA12, 0x00006674}, {0x0000FA15, 0x000051DE}, {0x0000FA16, 0x0000732A}, {0x0000FA17, 0x000076CA}, -{0x0000FA18, 0x0000793C}, {0x0000FA19, 0x0000795E}, {0x0000FA1A, 0x00007965}, {0x0000FA1B, 0x0000798F}, -{0x0000FA1C, 0x00009756}, {0x0000FA1D, 0x00007CBE}, {0x0000FA1E, 0x00007FBD}, {0x0000FA20, 0x00008612}, -{0x0000FA22, 0x00008AF8}, {0x0000FA25, 0x00009038}, {0x0000FA26, 0x000090FD}, {0x0000FA2A, 0x000098EF}, -{0x0000FA2B, 0x000098FC}, {0x0000FA2C, 0x00009928}, {0x0000FA2D, 0x00009DB4}, {0x0000FA2E, 0x000090DE}, -{0x0000FA2F, 0x000096B7}, {0x0000FA30, 0x00004FAE}, {0x0000FA31, 0x000050E7}, {0x0000FA32, 0x0000514D}, -{0x0000FA33, 0x000052C9}, {0x0000FA34, 0x000052E4}, {0x0000FA35, 0x00005351}, {0x0000FA36, 0x0000559D}, -{0x0000FA37, 0x00005606}, {0x0000FA38, 0x00005668}, {0x0000FA39, 0x00005840}, {0x0000FA3A, 0x000058A8}, -{0x0000FA3B, 0x00005C64}, {0x0000FA3C, 0x00005C6E}, {0x0000FA3D, 0x00006094}, {0x0000FA3E, 0x00006168}, -{0x0000FA3F, 0x0000618E}, {0x0000FA40, 0x000061F2}, {0x0000FA41, 0x0000654F}, {0x0000FA42, 0x000065E2}, -{0x0000FA43, 0x00006691}, {0x0000FA44, 0x00006885}, {0x0000FA45, 0x00006D77}, {0x0000FA46, 0x00006E1A}, -{0x0000FA47, 0x00006F22}, {0x0000FA48, 0x0000716E}, {0x0000FA49, 0x0000722B}, {0x0000FA4A, 0x00007422}, -{0x0000FA4B, 0x00007891}, {0x0000FA4C, 0x0000793E}, {0x0000FA4D, 0x00007949}, {0x0000FA4E, 0x00007948}, -{0x0000FA4F, 0x00007950}, {0x0000FA50, 0x00007956}, {0x0000FA51, 0x0000795D}, {0x0000FA52, 0x0000798D}, -{0x0000FA53, 0x0000798E}, {0x0000FA54, 0x00007A40}, {0x0000FA55, 0x00007A81}, {0x0000FA56, 0x00007BC0}, -{0x0000FA57, 0x00007DF4}, {0x0000FA58, 0x00007E09}, {0x0000FA59, 0x00007E41}, {0x0000FA5A, 0x00007F72}, -{0x0000FA5B, 0x00008005}, {0x0000FA5C, 0x000081ED}, {0x0000FA5D, 0x00008279}, {0x0000FA5E, 0x00008279}, -{0x0000FA5F, 0x00008457}, {0x0000FA60, 0x00008910}, {0x0000FA61, 0x00008996}, {0x0000FA62, 0x00008B01}, -{0x0000FA63, 0x00008B39}, {0x0000FA64, 0x00008CD3}, {0x0000FA65, 0x00008D08}, {0x0000FA66, 0x00008FB6}, -{0x0000FA67, 0x00009038}, {0x0000FA68, 0x000096E3}, {0x0000FA69, 0x000097FF}, {0x0000FA6A, 0x0000983B}, -{0x0000FA6B, 0x00006075}, {0x0000FA6C, 0x000242EE}, {0x0000FA6D, 0x00008218}, {0x0000FA70, 0x00004E26}, -{0x0000FA71, 0x000051B5}, {0x0000FA72, 0x00005168}, {0x0000FA73, 0x00004F80}, {0x0000FA74, 0x00005145}, -{0x0000FA75, 0x00005180}, {0x0000FA76, 0x000052C7}, {0x0000FA77, 0x000052FA}, {0x0000FA78, 0x0000559D}, -{0x0000FA79, 0x00005555}, {0x0000FA7A, 0x00005599}, {0x0000FA7B, 0x000055E2}, {0x0000FA7C, 0x0000585A}, -{0x0000FA7D, 0x000058B3}, {0x0000FA7E, 0x00005944}, {0x0000FA7F, 0x00005954}, {0x0000FA80, 0x00005A62}, -{0x0000FA81, 0x00005B28}, {0x0000FA82, 0x00005ED2}, {0x0000FA83, 0x00005ED9}, {0x0000FA84, 0x00005F69}, -{0x0000FA85, 0x00005FAD}, {0x0000FA86, 0x000060D8}, {0x0000FA87, 0x0000614E}, {0x0000FA88, 0x00006108}, -{0x0000FA89, 0x0000618E}, {0x0000FA8A, 0x00006160}, {0x0000FA8B, 0x000061F2}, {0x0000FA8C, 0x00006234}, -{0x0000FA8D, 0x000063C4}, {0x0000FA8E, 0x0000641C}, {0x0000FA8F, 0x00006452}, {0x0000FA90, 0x00006556}, -{0x0000FA91, 0x00006674}, {0x0000FA92, 0x00006717}, {0x0000FA93, 0x0000671B}, {0x0000FA94, 0x00006756}, -{0x0000FA95, 0x00006B79}, {0x0000FA96, 0x00006BBA}, {0x0000FA97, 0x00006D41}, {0x0000FA98, 0x00006EDB}, -{0x0000FA99, 0x00006ECB}, {0x0000FA9A, 0x00006F22}, {0x0000FA9B, 0x0000701E}, {0x0000FA9C, 0x0000716E}, -{0x0000FA9D, 0x000077A7}, {0x0000FA9E, 0x00007235}, {0x0000FA9F, 0x000072AF}, {0x0000FAA0, 0x0000732A}, -{0x0000FAA1, 0x00007471}, {0x0000FAA2, 0x00007506}, {0x0000FAA3, 0x0000753B}, {0x0000FAA4, 0x0000761D}, -{0x0000FAA5, 0x0000761F}, {0x0000FAA6, 0x000076CA}, {0x0000FAA7, 0x000076DB}, {0x0000FAA8, 0x000076F4}, -{0x0000FAA9, 0x0000774A}, {0x0000FAAA, 0x00007740}, {0x0000FAAB, 0x000078CC}, {0x0000FAAC, 0x00007AB1}, -{0x0000FAAD, 0x00007BC0}, {0x0000FAAE, 0x00007C7B}, {0x0000FAAF, 0x00007D5B}, {0x0000FAB0, 0x00007DF4}, -{0x0000FAB1, 0x00007F3E}, {0x0000FAB2, 0x00008005}, {0x0000FAB3, 0x00008352}, {0x0000FAB4, 0x000083EF}, -{0x0000FAB5, 0x00008779}, {0x0000FAB6, 0x00008941}, {0x0000FAB7, 0x00008986}, {0x0000FAB8, 0x00008996}, -{0x0000FAB9, 0x00008ABF}, {0x0000FABA, 0x00008AF8}, {0x0000FABB, 0x00008ACB}, {0x0000FABC, 0x00008B01}, -{0x0000FABD, 0x00008AFE}, {0x0000FABE, 0x00008AED}, {0x0000FABF, 0x00008B39}, {0x0000FAC0, 0x00008B8A}, -{0x0000FAC1, 0x00008D08}, {0x0000FAC2, 0x00008F38}, {0x0000FAC3, 0x00009072}, {0x0000FAC4, 0x00009199}, -{0x0000FAC5, 0x00009276}, {0x0000FAC6, 0x0000967C}, {0x0000FAC7, 0x000096E3}, {0x0000FAC8, 0x00009756}, -{0x0000FAC9, 0x000097DB}, {0x0000FACA, 0x000097FF}, {0x0000FACB, 0x0000980B}, {0x0000FACC, 0x0000983B}, -{0x0000FACD, 0x00009B12}, {0x0000FACE, 0x00009F9C}, {0x0000FACF, 0x0002284A}, {0x0000FAD0, 0x00022844}, -{0x0000FAD1, 0x000233D5}, {0x0000FAD2, 0x00003B9D}, {0x0000FAD3, 0x00004018}, {0x0000FAD4, 0x00004039}, -{0x0000FAD5, 0x00025249}, {0x0000FAD6, 0x00025CD0}, {0x0000FAD7, 0x00027ED3}, {0x0000FAD8, 0x00009F43}, -{0x0000FAD9, 0x00009F8E}, {0x0000FB1D, 0x000005D9}, {0x0000FB1D, 0x000005B4}, {0x0000FB1F, 0x000005F2}, -{0x0000FB1F, 0x000005B7}, {0x0000FB2A, 0x000005E9}, {0x0000FB2A, 0x000005C1}, {0x0000FB2B, 0x000005E9}, -{0x0000FB2B, 0x000005C2}, {0x0000FB2C, 0x000005E9}, {0x0000FB2C, 0x000005BC}, {0x0000FB2C, 0x000005C1}, -{0x0000FB2D, 0x000005E9}, {0x0000FB2D, 0x000005BC}, {0x0000FB2D, 0x000005C2}, {0x0000FB2E, 0x000005D0}, -{0x0000FB2E, 0x000005B7}, {0x0000FB2F, 0x000005D0}, {0x0000FB2F, 0x000005B8}, {0x0000FB30, 0x000005D0}, -{0x0000FB30, 0x000005BC}, {0x0000FB31, 0x000005D1}, {0x0000FB31, 0x000005BC}, {0x0000FB32, 0x000005D2}, -{0x0000FB32, 0x000005BC}, {0x0000FB33, 0x000005D3}, {0x0000FB33, 0x000005BC}, {0x0000FB34, 0x000005D4}, -{0x0000FB34, 0x000005BC}, {0x0000FB35, 0x000005D5}, {0x0000FB35, 0x000005BC}, {0x0000FB36, 0x000005D6}, -{0x0000FB36, 0x000005BC}, {0x0000FB38, 0x000005D8}, {0x0000FB38, 0x000005BC}, {0x0000FB39, 0x000005D9}, -{0x0000FB39, 0x000005BC}, {0x0000FB3A, 0x000005DA}, {0x0000FB3A, 0x000005BC}, {0x0000FB3B, 0x000005DB}, -{0x0000FB3B, 0x000005BC}, {0x0000FB3C, 0x000005DC}, {0x0000FB3C, 0x000005BC}, {0x0000FB3E, 0x000005DE}, -{0x0000FB3E, 0x000005BC}, {0x0000FB40, 0x000005E0}, {0x0000FB40, 0x000005BC}, {0x0000FB41, 0x000005E1}, -{0x0000FB41, 0x000005BC}, {0x0000FB43, 0x000005E3}, {0x0000FB43, 0x000005BC}, {0x0000FB44, 0x000005E4}, -{0x0000FB44, 0x000005BC}, {0x0000FB46, 0x000005E6}, {0x0000FB46, 0x000005BC}, {0x0000FB47, 0x000005E7}, -{0x0000FB47, 0x000005BC}, {0x0000FB48, 0x000005E8}, {0x0000FB48, 0x000005BC}, {0x0000FB49, 0x000005E9}, -{0x0000FB49, 0x000005BC}, {0x0000FB4A, 0x000005EA}, {0x0000FB4A, 0x000005BC}, {0x0000FB4B, 0x000005D5}, -{0x0000FB4B, 0x000005B9}, {0x0000FB4C, 0x000005D1}, {0x0000FB4C, 0x000005BF}, {0x0000FB4D, 0x000005DB}, -{0x0000FB4D, 0x000005BF}, {0x0000FB4E, 0x000005E4}, {0x0000FB4E, 0x000005BF}, {0x0001109A, 0x00011099}, -{0x0001109A, 0x000110BA}, {0x0001109C, 0x0001109B}, {0x0001109C, 0x000110BA}, {0x000110AB, 0x000110A5}, -{0x000110AB, 0x000110BA}, {0x0001112E, 0x00011131}, {0x0001112E, 0x00011127}, {0x0001112F, 0x00011132}, -{0x0001112F, 0x00011127}, {0x0001134B, 0x00011347}, {0x0001134B, 0x0001133E}, {0x0001134C, 0x00011347}, -{0x0001134C, 0x00011357}, {0x000114BB, 0x000114B9}, {0x000114BB, 0x000114BA}, {0x000114BC, 0x000114B9}, -{0x000114BC, 0x000114B0}, {0x000114BE, 0x000114B9}, {0x000114BE, 0x000114BD}, {0x000115BA, 0x000115B8}, -{0x000115BA, 0x000115AF}, {0x000115BB, 0x000115B9}, {0x000115BB, 0x000115AF}, {0x0001D15E, 0x0001D157}, -{0x0001D15E, 0x0001D165}, {0x0001D15F, 0x0001D158}, {0x0001D15F, 0x0001D165}, {0x0001D160, 0x0001D158}, -{0x0001D160, 0x0001D165}, {0x0001D160, 0x0001D16E}, {0x0001D161, 0x0001D158}, {0x0001D161, 0x0001D165}, -{0x0001D161, 0x0001D16F}, {0x0001D162, 0x0001D158}, {0x0001D162, 0x0001D165}, {0x0001D162, 0x0001D170}, -{0x0001D163, 0x0001D158}, {0x0001D163, 0x0001D165}, {0x0001D163, 0x0001D171}, {0x0001D164, 0x0001D158}, -{0x0001D164, 0x0001D165}, {0x0001D164, 0x0001D172}, {0x0001D1BB, 0x0001D1B9}, {0x0001D1BB, 0x0001D165}, -{0x0001D1BC, 0x0001D1BA}, {0x0001D1BC, 0x0001D165}, {0x0001D1BD, 0x0001D1B9}, {0x0001D1BD, 0x0001D165}, -{0x0001D1BD, 0x0001D16E}, {0x0001D1BE, 0x0001D1BA}, {0x0001D1BE, 0x0001D165}, {0x0001D1BE, 0x0001D16E}, -{0x0001D1BF, 0x0001D1B9}, {0x0001D1BF, 0x0001D165}, {0x0001D1BF, 0x0001D16F}, {0x0001D1C0, 0x0001D1BA}, -{0x0001D1C0, 0x0001D165}, {0x0001D1C0, 0x0001D16F}, {0x0002F800, 0x00004E3D}, {0x0002F801, 0x00004E38}, -{0x0002F802, 0x00004E41}, {0x0002F803, 0x00020122}, {0x0002F804, 0x00004F60}, {0x0002F805, 0x00004FAE}, -{0x0002F806, 0x00004FBB}, {0x0002F807, 0x00005002}, {0x0002F808, 0x0000507A}, {0x0002F809, 0x00005099}, -{0x0002F80A, 0x000050E7}, {0x0002F80B, 0x000050CF}, {0x0002F80C, 0x0000349E}, {0x0002F80D, 0x0002063A}, -{0x0002F80E, 0x0000514D}, {0x0002F80F, 0x00005154}, {0x0002F810, 0x00005164}, {0x0002F811, 0x00005177}, -{0x0002F812, 0x0002051C}, {0x0002F813, 0x000034B9}, {0x0002F814, 0x00005167}, {0x0002F815, 0x0000518D}, -{0x0002F816, 0x0002054B}, {0x0002F817, 0x00005197}, {0x0002F818, 0x000051A4}, {0x0002F819, 0x00004ECC}, -{0x0002F81A, 0x000051AC}, {0x0002F81B, 0x000051B5}, {0x0002F81C, 0x000291DF}, {0x0002F81D, 0x000051F5}, -{0x0002F81E, 0x00005203}, {0x0002F81F, 0x000034DF}, {0x0002F820, 0x0000523B}, {0x0002F821, 0x00005246}, -{0x0002F822, 0x00005272}, {0x0002F823, 0x00005277}, {0x0002F824, 0x00003515}, {0x0002F825, 0x000052C7}, -{0x0002F826, 0x000052C9}, {0x0002F827, 0x000052E4}, {0x0002F828, 0x000052FA}, {0x0002F829, 0x00005305}, -{0x0002F82A, 0x00005306}, {0x0002F82B, 0x00005317}, {0x0002F82C, 0x00005349}, {0x0002F82D, 0x00005351}, -{0x0002F82E, 0x0000535A}, {0x0002F82F, 0x00005373}, {0x0002F830, 0x0000537D}, {0x0002F831, 0x0000537F}, -{0x0002F832, 0x0000537F}, {0x0002F833, 0x0000537F}, {0x0002F834, 0x00020A2C}, {0x0002F835, 0x00007070}, -{0x0002F836, 0x000053CA}, {0x0002F837, 0x000053DF}, {0x0002F838, 0x00020B63}, {0x0002F839, 0x000053EB}, -{0x0002F83A, 0x000053F1}, {0x0002F83B, 0x00005406}, {0x0002F83C, 0x0000549E}, {0x0002F83D, 0x00005438}, -{0x0002F83E, 0x00005448}, {0x0002F83F, 0x00005468}, {0x0002F840, 0x000054A2}, {0x0002F841, 0x000054F6}, -{0x0002F842, 0x00005510}, {0x0002F843, 0x00005553}, {0x0002F844, 0x00005563}, {0x0002F845, 0x00005584}, -{0x0002F846, 0x00005584}, {0x0002F847, 0x00005599}, {0x0002F848, 0x000055AB}, {0x0002F849, 0x000055B3}, -{0x0002F84A, 0x000055C2}, {0x0002F84B, 0x00005716}, {0x0002F84C, 0x00005606}, {0x0002F84D, 0x00005717}, -{0x0002F84E, 0x00005651}, {0x0002F84F, 0x00005674}, {0x0002F850, 0x00005207}, {0x0002F851, 0x000058EE}, -{0x0002F852, 0x000057CE}, {0x0002F853, 0x000057F4}, {0x0002F854, 0x0000580D}, {0x0002F855, 0x0000578B}, -{0x0002F856, 0x00005832}, {0x0002F857, 0x00005831}, {0x0002F858, 0x000058AC}, {0x0002F859, 0x000214E4}, -{0x0002F85A, 0x000058F2}, {0x0002F85B, 0x000058F7}, {0x0002F85C, 0x00005906}, {0x0002F85D, 0x0000591A}, -{0x0002F85E, 0x00005922}, {0x0002F85F, 0x00005962}, {0x0002F860, 0x000216A8}, {0x0002F861, 0x000216EA}, -{0x0002F862, 0x000059EC}, {0x0002F863, 0x00005A1B}, {0x0002F864, 0x00005A27}, {0x0002F865, 0x000059D8}, -{0x0002F866, 0x00005A66}, {0x0002F867, 0x000036EE}, {0x0002F868, 0x000036FC}, {0x0002F869, 0x00005B08}, -{0x0002F86A, 0x00005B3E}, {0x0002F86B, 0x00005B3E}, {0x0002F86C, 0x000219C8}, {0x0002F86D, 0x00005BC3}, -{0x0002F86E, 0x00005BD8}, {0x0002F86F, 0x00005BE7}, {0x0002F870, 0x00005BF3}, {0x0002F871, 0x00021B18}, -{0x0002F872, 0x00005BFF}, {0x0002F873, 0x00005C06}, {0x0002F874, 0x00005F53}, {0x0002F875, 0x00005C22}, -{0x0002F876, 0x00003781}, {0x0002F877, 0x00005C60}, {0x0002F878, 0x00005C6E}, {0x0002F879, 0x00005CC0}, -{0x0002F87A, 0x00005C8D}, {0x0002F87B, 0x00021DE4}, {0x0002F87C, 0x00005D43}, {0x0002F87D, 0x00021DE6}, -{0x0002F87E, 0x00005D6E}, {0x0002F87F, 0x00005D6B}, {0x0002F880, 0x00005D7C}, {0x0002F881, 0x00005DE1}, -{0x0002F882, 0x00005DE2}, {0x0002F883, 0x0000382F}, {0x0002F884, 0x00005DFD}, {0x0002F885, 0x00005E28}, -{0x0002F886, 0x00005E3D}, {0x0002F887, 0x00005E69}, {0x0002F888, 0x00003862}, {0x0002F889, 0x00022183}, -{0x0002F88A, 0x0000387C}, {0x0002F88B, 0x00005EB0}, {0x0002F88C, 0x00005EB3}, {0x0002F88D, 0x00005EB6}, -{0x0002F88E, 0x00005ECA}, {0x0002F88F, 0x0002A392}, {0x0002F890, 0x00005EFE}, {0x0002F891, 0x00022331}, -{0x0002F892, 0x00022331}, {0x0002F893, 0x00008201}, {0x0002F894, 0x00005F22}, {0x0002F895, 0x00005F22}, -{0x0002F896, 0x000038C7}, {0x0002F897, 0x000232B8}, {0x0002F898, 0x000261DA}, {0x0002F899, 0x00005F62}, -{0x0002F89A, 0x00005F6B}, {0x0002F89B, 0x000038E3}, {0x0002F89C, 0x00005F9A}, {0x0002F89D, 0x00005FCD}, -{0x0002F89E, 0x00005FD7}, {0x0002F89F, 0x00005FF9}, {0x0002F8A0, 0x00006081}, {0x0002F8A1, 0x0000393A}, -{0x0002F8A2, 0x0000391C}, {0x0002F8A3, 0x00006094}, {0x0002F8A4, 0x000226D4}, {0x0002F8A5, 0x000060C7}, -{0x0002F8A6, 0x00006148}, {0x0002F8A7, 0x0000614C}, {0x0002F8A8, 0x0000614E}, {0x0002F8A9, 0x0000614C}, -{0x0002F8AA, 0x0000617A}, {0x0002F8AB, 0x0000618E}, {0x0002F8AC, 0x000061B2}, {0x0002F8AD, 0x000061A4}, -{0x0002F8AE, 0x000061AF}, {0x0002F8AF, 0x000061DE}, {0x0002F8B0, 0x000061F2}, {0x0002F8B1, 0x000061F6}, -{0x0002F8B2, 0x00006210}, {0x0002F8B3, 0x0000621B}, {0x0002F8B4, 0x0000625D}, {0x0002F8B5, 0x000062B1}, -{0x0002F8B6, 0x000062D4}, {0x0002F8B7, 0x00006350}, {0x0002F8B8, 0x00022B0C}, {0x0002F8B9, 0x0000633D}, -{0x0002F8BA, 0x000062FC}, {0x0002F8BB, 0x00006368}, {0x0002F8BC, 0x00006383}, {0x0002F8BD, 0x000063E4}, -{0x0002F8BE, 0x00022BF1}, {0x0002F8BF, 0x00006422}, {0x0002F8C0, 0x000063C5}, {0x0002F8C1, 0x000063A9}, -{0x0002F8C2, 0x00003A2E}, {0x0002F8C3, 0x00006469}, {0x0002F8C4, 0x0000647E}, {0x0002F8C5, 0x0000649D}, -{0x0002F8C6, 0x00006477}, {0x0002F8C7, 0x00003A6C}, {0x0002F8C8, 0x0000654F}, {0x0002F8C9, 0x0000656C}, -{0x0002F8CA, 0x0002300A}, {0x0002F8CB, 0x000065E3}, {0x0002F8CC, 0x000066F8}, {0x0002F8CD, 0x00006649}, -{0x0002F8CE, 0x00003B19}, {0x0002F8CF, 0x00006691}, {0x0002F8D0, 0x00003B08}, {0x0002F8D1, 0x00003AE4}, -{0x0002F8D2, 0x00005192}, {0x0002F8D3, 0x00005195}, {0x0002F8D4, 0x00006700}, {0x0002F8D5, 0x0000669C}, -{0x0002F8D6, 0x000080AD}, {0x0002F8D7, 0x000043D9}, {0x0002F8D8, 0x00006717}, {0x0002F8D9, 0x0000671B}, -{0x0002F8DA, 0x00006721}, {0x0002F8DB, 0x0000675E}, {0x0002F8DC, 0x00006753}, {0x0002F8DD, 0x000233C3}, -{0x0002F8DE, 0x00003B49}, {0x0002F8DF, 0x000067FA}, {0x0002F8E0, 0x00006785}, {0x0002F8E1, 0x00006852}, -{0x0002F8E2, 0x00006885}, {0x0002F8E3, 0x0002346D}, {0x0002F8E4, 0x0000688E}, {0x0002F8E5, 0x0000681F}, -{0x0002F8E6, 0x00006914}, {0x0002F8E7, 0x00003B9D}, {0x0002F8E8, 0x00006942}, {0x0002F8E9, 0x000069A3}, -{0x0002F8EA, 0x000069EA}, {0x0002F8EB, 0x00006AA8}, {0x0002F8EC, 0x000236A3}, {0x0002F8ED, 0x00006ADB}, -{0x0002F8EE, 0x00003C18}, {0x0002F8EF, 0x00006B21}, {0x0002F8F0, 0x000238A7}, {0x0002F8F1, 0x00006B54}, -{0x0002F8F2, 0x00003C4E}, {0x0002F8F3, 0x00006B72}, {0x0002F8F4, 0x00006B9F}, {0x0002F8F5, 0x00006BBA}, -{0x0002F8F6, 0x00006BBB}, {0x0002F8F7, 0x00023A8D}, {0x0002F8F8, 0x00021D0B}, {0x0002F8F9, 0x00023AFA}, -{0x0002F8FA, 0x00006C4E}, {0x0002F8FB, 0x00023CBC}, {0x0002F8FC, 0x00006CBF}, {0x0002F8FD, 0x00006CCD}, -{0x0002F8FE, 0x00006C67}, {0x0002F8FF, 0x00006D16}, {0x0002F900, 0x00006D3E}, {0x0002F901, 0x00006D77}, -{0x0002F902, 0x00006D41}, {0x0002F903, 0x00006D69}, {0x0002F904, 0x00006D78}, {0x0002F905, 0x00006D85}, -{0x0002F906, 0x00023D1E}, {0x0002F907, 0x00006D34}, {0x0002F908, 0x00006E2F}, {0x0002F909, 0x00006E6E}, -{0x0002F90A, 0x00003D33}, {0x0002F90B, 0x00006ECB}, {0x0002F90C, 0x00006EC7}, {0x0002F90D, 0x00023ED1}, -{0x0002F90E, 0x00006DF9}, {0x0002F90F, 0x00006F6E}, {0x0002F910, 0x00023F5E}, {0x0002F911, 0x00023F8E}, -{0x0002F912, 0x00006FC6}, {0x0002F913, 0x00007039}, {0x0002F914, 0x0000701E}, {0x0002F915, 0x0000701B}, -{0x0002F916, 0x00003D96}, {0x0002F917, 0x0000704A}, {0x0002F918, 0x0000707D}, {0x0002F919, 0x00007077}, -{0x0002F91A, 0x000070AD}, {0x0002F91B, 0x00020525}, {0x0002F91C, 0x00007145}, {0x0002F91D, 0x00024263}, -{0x0002F91E, 0x0000719C}, {0x0002F91F, 0x000243AB}, {0x0002F920, 0x00007228}, {0x0002F921, 0x00007235}, -{0x0002F922, 0x00007250}, {0x0002F923, 0x00024608}, {0x0002F924, 0x00007280}, {0x0002F925, 0x00007295}, -{0x0002F926, 0x00024735}, {0x0002F927, 0x00024814}, {0x0002F928, 0x0000737A}, {0x0002F929, 0x0000738B}, -{0x0002F92A, 0x00003EAC}, {0x0002F92B, 0x000073A5}, {0x0002F92C, 0x00003EB8}, {0x0002F92D, 0x00003EB8}, -{0x0002F92E, 0x00007447}, {0x0002F92F, 0x0000745C}, {0x0002F930, 0x00007471}, {0x0002F931, 0x00007485}, -{0x0002F932, 0x000074CA}, {0x0002F933, 0x00003F1B}, {0x0002F934, 0x00007524}, {0x0002F935, 0x00024C36}, -{0x0002F936, 0x0000753E}, {0x0002F937, 0x00024C92}, {0x0002F938, 0x00007570}, {0x0002F939, 0x0002219F}, -{0x0002F93A, 0x00007610}, {0x0002F93B, 0x00024FA1}, {0x0002F93C, 0x00024FB8}, {0x0002F93D, 0x00025044}, -{0x0002F93E, 0x00003FFC}, {0x0002F93F, 0x00004008}, {0x0002F940, 0x000076F4}, {0x0002F941, 0x000250F3}, -{0x0002F942, 0x000250F2}, {0x0002F943, 0x00025119}, {0x0002F944, 0x00025133}, {0x0002F945, 0x0000771E}, -{0x0002F946, 0x0000771F}, {0x0002F947, 0x0000771F}, {0x0002F948, 0x0000774A}, {0x0002F949, 0x00004039}, -{0x0002F94A, 0x0000778B}, {0x0002F94B, 0x00004046}, {0x0002F94C, 0x00004096}, {0x0002F94D, 0x0002541D}, -{0x0002F94E, 0x0000784E}, {0x0002F94F, 0x0000788C}, {0x0002F950, 0x000078CC}, {0x0002F951, 0x000040E3}, -{0x0002F952, 0x00025626}, {0x0002F953, 0x00007956}, {0x0002F954, 0x0002569A}, {0x0002F955, 0x000256C5}, -{0x0002F956, 0x0000798F}, {0x0002F957, 0x000079EB}, {0x0002F958, 0x0000412F}, {0x0002F959, 0x00007A40}, -{0x0002F95A, 0x00007A4A}, {0x0002F95B, 0x00007A4F}, {0x0002F95C, 0x0002597C}, {0x0002F95D, 0x00025AA7}, -{0x0002F95E, 0x00025AA7}, {0x0002F95F, 0x00007AEE}, {0x0002F960, 0x00004202}, {0x0002F961, 0x00025BAB}, -{0x0002F962, 0x00007BC6}, {0x0002F963, 0x00007BC9}, {0x0002F964, 0x00004227}, {0x0002F965, 0x00025C80}, -{0x0002F966, 0x00007CD2}, {0x0002F967, 0x000042A0}, {0x0002F968, 0x00007CE8}, {0x0002F969, 0x00007CE3}, -{0x0002F96A, 0x00007D00}, {0x0002F96B, 0x00025F86}, {0x0002F96C, 0x00007D63}, {0x0002F96D, 0x00004301}, -{0x0002F96E, 0x00007DC7}, {0x0002F96F, 0x00007E02}, {0x0002F970, 0x00007E45}, {0x0002F971, 0x00004334}, -{0x0002F972, 0x00026228}, {0x0002F973, 0x00026247}, {0x0002F974, 0x00004359}, {0x0002F975, 0x000262D9}, -{0x0002F976, 0x00007F7A}, {0x0002F977, 0x0002633E}, {0x0002F978, 0x00007F95}, {0x0002F979, 0x00007FFA}, -{0x0002F97A, 0x00008005}, {0x0002F97B, 0x000264DA}, {0x0002F97C, 0x00026523}, {0x0002F97D, 0x00008060}, -{0x0002F97E, 0x000265A8}, {0x0002F97F, 0x00008070}, {0x0002F980, 0x0002335F}, {0x0002F981, 0x000043D5}, -{0x0002F982, 0x000080B2}, {0x0002F983, 0x00008103}, {0x0002F984, 0x0000440B}, {0x0002F985, 0x0000813E}, -{0x0002F986, 0x00005AB5}, {0x0002F987, 0x000267A7}, {0x0002F988, 0x000267B5}, {0x0002F989, 0x00023393}, -{0x0002F98A, 0x0002339C}, {0x0002F98B, 0x00008201}, {0x0002F98C, 0x00008204}, {0x0002F98D, 0x00008F9E}, -{0x0002F98E, 0x0000446B}, {0x0002F98F, 0x00008291}, {0x0002F990, 0x0000828B}, {0x0002F991, 0x0000829D}, -{0x0002F992, 0x000052B3}, {0x0002F993, 0x000082B1}, {0x0002F994, 0x000082B3}, {0x0002F995, 0x000082BD}, -{0x0002F996, 0x000082E6}, {0x0002F997, 0x00026B3C}, {0x0002F998, 0x000082E5}, {0x0002F999, 0x0000831D}, -{0x0002F99A, 0x00008363}, {0x0002F99B, 0x000083AD}, {0x0002F99C, 0x00008323}, {0x0002F99D, 0x000083BD}, -{0x0002F99E, 0x000083E7}, {0x0002F99F, 0x00008457}, {0x0002F9A0, 0x00008353}, {0x0002F9A1, 0x000083CA}, -{0x0002F9A2, 0x000083CC}, {0x0002F9A3, 0x000083DC}, {0x0002F9A4, 0x00026C36}, {0x0002F9A5, 0x00026D6B}, -{0x0002F9A6, 0x00026CD5}, {0x0002F9A7, 0x0000452B}, {0x0002F9A8, 0x000084F1}, {0x0002F9A9, 0x000084F3}, -{0x0002F9AA, 0x00008516}, {0x0002F9AB, 0x000273CA}, {0x0002F9AC, 0x00008564}, {0x0002F9AD, 0x00026F2C}, -{0x0002F9AE, 0x0000455D}, {0x0002F9AF, 0x00004561}, {0x0002F9B0, 0x00026FB1}, {0x0002F9B1, 0x000270D2}, -{0x0002F9B2, 0x0000456B}, {0x0002F9B3, 0x00008650}, {0x0002F9B4, 0x0000865C}, {0x0002F9B5, 0x00008667}, -{0x0002F9B6, 0x00008669}, {0x0002F9B7, 0x000086A9}, {0x0002F9B8, 0x00008688}, {0x0002F9B9, 0x0000870E}, -{0x0002F9BA, 0x000086E2}, {0x0002F9BB, 0x00008779}, {0x0002F9BC, 0x00008728}, {0x0002F9BD, 0x0000876B}, -{0x0002F9BE, 0x00008786}, {0x0002F9BF, 0x000045D7}, {0x0002F9C0, 0x000087E1}, {0x0002F9C1, 0x00008801}, -{0x0002F9C2, 0x000045F9}, {0x0002F9C3, 0x00008860}, {0x0002F9C4, 0x00008863}, {0x0002F9C5, 0x00027667}, -{0x0002F9C6, 0x000088D7}, {0x0002F9C7, 0x000088DE}, {0x0002F9C8, 0x00004635}, {0x0002F9C9, 0x000088FA}, -{0x0002F9CA, 0x000034BB}, {0x0002F9CB, 0x000278AE}, {0x0002F9CC, 0x00027966}, {0x0002F9CD, 0x000046BE}, -{0x0002F9CE, 0x000046C7}, {0x0002F9CF, 0x00008AA0}, {0x0002F9D0, 0x00008AED}, {0x0002F9D1, 0x00008B8A}, -{0x0002F9D2, 0x00008C55}, {0x0002F9D3, 0x00027CA8}, {0x0002F9D4, 0x00008CAB}, {0x0002F9D5, 0x00008CC1}, -{0x0002F9D6, 0x00008D1B}, {0x0002F9D7, 0x00008D77}, {0x0002F9D8, 0x00027F2F}, {0x0002F9D9, 0x00020804}, -{0x0002F9DA, 0x00008DCB}, {0x0002F9DB, 0x00008DBC}, {0x0002F9DC, 0x00008DF0}, {0x0002F9DD, 0x000208DE}, -{0x0002F9DE, 0x00008ED4}, {0x0002F9DF, 0x00008F38}, {0x0002F9E0, 0x000285D2}, {0x0002F9E1, 0x000285ED}, -{0x0002F9E2, 0x00009094}, {0x0002F9E3, 0x000090F1}, {0x0002F9E4, 0x00009111}, {0x0002F9E5, 0x0002872E}, -{0x0002F9E6, 0x0000911B}, {0x0002F9E7, 0x00009238}, {0x0002F9E8, 0x000092D7}, {0x0002F9E9, 0x000092D8}, -{0x0002F9EA, 0x0000927C}, {0x0002F9EB, 0x000093F9}, {0x0002F9EC, 0x00009415}, {0x0002F9ED, 0x00028BFA}, -{0x0002F9EE, 0x0000958B}, {0x0002F9EF, 0x00004995}, {0x0002F9F0, 0x000095B7}, {0x0002F9F1, 0x00028D77}, -{0x0002F9F2, 0x000049E6}, {0x0002F9F3, 0x000096C3}, {0x0002F9F4, 0x00005DB2}, {0x0002F9F5, 0x00009723}, -{0x0002F9F6, 0x00029145}, {0x0002F9F7, 0x0002921A}, {0x0002F9F8, 0x00004A6E}, {0x0002F9F9, 0x00004A76}, -{0x0002F9FA, 0x000097E0}, {0x0002F9FB, 0x0002940A}, {0x0002F9FC, 0x00004AB2}, {0x0002F9FD, 0x00029496}, -{0x0002F9FE, 0x0000980B}, {0x0002F9FF, 0x0000980B}, {0x0002FA00, 0x00009829}, {0x0002FA01, 0x000295B6}, -{0x0002FA02, 0x000098E2}, {0x0002FA03, 0x00004B33}, {0x0002FA04, 0x00009929}, {0x0002FA05, 0x000099A7}, -{0x0002FA06, 0x000099C2}, {0x0002FA07, 0x000099FE}, {0x0002FA08, 0x00004BCE}, {0x0002FA09, 0x00029B30}, -{0x0002FA0A, 0x00009B12}, {0x0002FA0B, 0x00009C40}, {0x0002FA0C, 0x00009CFD}, {0x0002FA0D, 0x00004CCE}, -{0x0002FA0E, 0x00004CED}, {0x0002FA0F, 0x00009D67}, {0x0002FA10, 0x0002A0CE}, {0x0002FA11, 0x00004CF8}, -{0x0002FA12, 0x0002A105}, {0x0002FA13, 0x0002A20E}, {0x0002FA14, 0x0002A291}, {0x0002FA15, 0x00009EBB}, -{0x0002FA16, 0x00004D56}, {0x0002FA17, 0x00009EF9}, {0x0002FA18, 0x00009EFE}, {0x0002FA19, 0x00009F05}, -{0x0002FA1A, 0x00009F0F}, {0x0002FA1B, 0x00009F16}, {0x0002FA1D, 0x0002A600}, -}; diff --git a/unicode-data.h b/unicode-data.h index 3cccf2068..e27fe1770 100644 --- a/unicode-data.h +++ b/unicode-data.h @@ -1,17 +1,20 @@ #pragma once #include -#include -#include #include +#include +#include -extern const std::vector> unicode_ranges_number; -extern const std::vector> unicode_ranges_letter; -extern const std::vector> unicode_ranges_separator; -extern const std::vector> unicode_ranges_whitespace; -extern const std::vector> unicode_ranges_accent_mark; -extern const std::vector> unicode_ranges_punctuation; -extern const std::vector> unicode_ranges_symbol; -extern const std::vector> unicode_ranges_control; -extern const std::multimap unicode_map_nfd; -extern const std::map unicode_map_lowercase; +struct range_nfd { + uint32_t first; + uint32_t last; + uint32_t nfd; +}; + +static const uint32_t MAX_CODEPOINTS = 0x110000; + +extern const std::vector> unicode_ranges_flags; +extern const std::unordered_set unicode_set_whitespace; +extern const std::unordered_map unicode_map_lowercase; +extern const std::unordered_map unicode_map_uppercase; +extern const std::vector unicode_ranges_nfd; diff --git a/unicode.cpp b/unicode.cpp index ca03c49d3..056a4c741 100644 --- a/unicode.cpp +++ b/unicode.cpp @@ -1,4 +1,4 @@ -#include "unicode.h" +#include "unicode.h" #include "unicode-data.h" #include @@ -109,57 +109,49 @@ static uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) // return result; //} -static std::unordered_map unicode_cpt_type_map() { - std::unordered_map cpt_types; - for (auto p : unicode_ranges_number) { - for (auto i = p.first; i <= p.second; ++i) { - cpt_types[i] = CODEPOINT_TYPE_NUMBER; +static std::vector unicode_cpt_flags_array() { + std::vector cpt_flags(MAX_CODEPOINTS, codepoint_flags::UNDEFINED); + + assert (unicode_ranges_flags.front().first == 0); + assert (unicode_ranges_flags.back().first == MAX_CODEPOINTS); + for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) { + const auto range_ini = unicode_ranges_flags[i-1]; // codepoint_ini, flags + const auto range_end = unicode_ranges_flags[i]; // codepoint_end, flags + for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) { + cpt_flags[cpt] = range_ini.second; } } - for (auto p : unicode_ranges_letter) { - for (auto i = p.first; i <= p.second; ++i) { - cpt_types[i] = CODEPOINT_TYPE_LETTER; - } + + for (auto cpt : unicode_set_whitespace) { + cpt_flags[cpt].is_whitespace = true; } - for (auto p : unicode_ranges_separator) { - for (auto i = p.first; i <= p.second; ++i) { - cpt_types[i] = CODEPOINT_TYPE_SEPARATOR; - } + + for (auto p : unicode_map_lowercase) { + cpt_flags[p.second].is_lowercase = true; } - for (auto p : unicode_ranges_accent_mark) { - for (auto i = p.first; i <= p.second; ++i) { - cpt_types[i] = CODEPOINT_TYPE_ACCENT_MARK; - } + + for (auto p : unicode_map_uppercase) { + cpt_flags[p.second].is_uppercase = true; } - for (auto p : unicode_ranges_punctuation) { - for (auto i = p.first; i <= p.second; ++i) { - cpt_types[i] = CODEPOINT_TYPE_PUNCTUATION; - } + + for (auto &range : unicode_ranges_nfd) { // start, last, nfd + cpt_flags[range.nfd].is_nfd = true; } - for (auto p : unicode_ranges_symbol) { - for (auto i = p.first; i <= p.second; ++i) { - cpt_types[i] = CODEPOINT_TYPE_SYMBOL; - } - } - for (auto p : unicode_ranges_control) { - for (auto i = p.first; i <= p.second; ++i) { - cpt_types[i] = CODEPOINT_TYPE_CONTROL; - } - } - return cpt_types; + + return cpt_flags; } static std::unordered_map unicode_byte_to_utf8_map() { std::unordered_map map; - for (int ch = u'!'; ch <= u'~'; ++ch) { + for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~' assert(0 <= ch && ch < 256); map[ch] = unicode_cpt_to_utf8(ch); } - for (int ch = u'¡'; ch <= u'¬'; ++ch) { + for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬' assert(0 <= ch && ch < 256); map[ch] = unicode_cpt_to_utf8(ch); } - for (int ch = u'®'; ch <= u'ÿ'; ++ch) { + for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ' assert(0 <= ch && ch < 256); map[ch] = unicode_cpt_to_utf8(ch); } @@ -175,15 +167,15 @@ static std::unordered_map unicode_byte_to_utf8_map() { static std::unordered_map unicode_utf8_to_byte_map() { std::unordered_map map; - for (int ch = u'!'; ch <= u'~'; ++ch) { + for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~' assert(0 <= ch && ch < 256); map[unicode_cpt_to_utf8(ch)] = ch; } - for (int ch = u'¡'; ch <= u'¬'; ++ch) { + for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬' assert(0 <= ch && ch < 256); map[unicode_cpt_to_utf8(ch)] = ch; } - for (int ch = u'®'; ch <= u'ÿ'; ++ch) { + for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ' assert(0 <= ch && ch < 256); map[unicode_cpt_to_utf8(ch)] = ch; } @@ -238,8 +230,9 @@ static std::vector unicode_regex_split_custom_gpt2(const std::string & t return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : 0; }; - auto _get_cpt_type = [&] (const size_t pos) -> int { - return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_type(cpts[pos]) : CODEPOINT_TYPE_UNIDENTIFIED; + auto _get_flags = [&] (const size_t pos) -> codepoint_flags { + static const codepoint_flags undef(codepoint_flags::UNDEFINED); + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : undef; }; size_t _prev_end = offset_ini; @@ -261,7 +254,7 @@ static std::vector unicode_regex_split_custom_gpt2(const std::string & t for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { const char32_t cpt = _get_cpt(pos); - const int cpt_type = _get_cpt_type(pos); + const auto flags = _get_flags(pos); // regex: 's|'t|'re|'ve|'m|'ll|'d if (cpt == '\'' && pos+1 < offset_end) { @@ -281,39 +274,37 @@ static std::vector unicode_regex_split_custom_gpt2(const std::string & t } } - char32_t cpt2 = (cpt == ' ' ? _get_cpt(pos+1) : cpt); - int cpt2_type = (cpt == ' ' ? _get_cpt_type(pos+1) : cpt_type); + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); // regex: ?\p{L}+ - if (cpt2_type == CODEPOINT_TYPE_LETTER) { + if (flags2.is_letter) { pos += (cpt == ' '); - while (cpt2_type == CODEPOINT_TYPE_LETTER) { - cpt2_type = _get_cpt_type(++pos); + while (flags2.is_letter) { + flags2 = _get_flags(++pos); } _add_token(pos); continue; } // regex: ?\p{N}+ - if (cpt2_type == CODEPOINT_TYPE_NUMBER) { + if (flags2.is_number) { pos += (cpt == ' '); - while (cpt2_type == CODEPOINT_TYPE_NUMBER) { - cpt2_type = _get_cpt_type(++pos); + while (flags2.is_number) { + flags2 = _get_flags(++pos); } _add_token(pos); continue; } // regex: ?[^\s\p{L}\p{N}]+ - if (!unicode_cpt_is_whitespace(cpt2) && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_NUMBER && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) { + if (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number || flags2.is_undefined)) { pos += (cpt == ' '); - while (!unicode_cpt_is_whitespace(cpt2) && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_NUMBER && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) { - cpt2_type = _get_cpt_type(++pos); - cpt2 = _get_cpt(pos); + while (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number || flags2.is_undefined)) { + flags2 = _get_flags(++pos); } _add_token(pos); continue; } size_t num_whitespaces = 0; - while (unicode_cpt_is_whitespace(_get_cpt(pos+num_whitespaces))) { + while (_get_flags(pos+num_whitespaces).is_whitespace) { num_whitespaces++; } @@ -357,8 +348,9 @@ static std::vector unicode_regex_split_custom_llama3(const std::string & return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : 0; }; - auto _get_cpt_type = [&] (const size_t pos) -> int { - return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_type(cpts[pos]) : CODEPOINT_TYPE_UNIDENTIFIED; + auto _get_flags = [&] (const size_t pos) -> codepoint_flags { + static const codepoint_flags undef(codepoint_flags::UNDEFINED); + return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags(cpts[pos]) : undef; }; size_t _prev_end = offset_ini; @@ -380,7 +372,7 @@ static std::vector unicode_regex_split_custom_llama3(const std::string & for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { const char32_t cpt = _get_cpt(pos); - const int cpt_type = _get_cpt_type(pos); + const auto flags = _get_flags(pos); // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive if (cpt == '\'' && pos+1 < offset_end) { @@ -401,10 +393,10 @@ static std::vector unicode_regex_split_custom_llama3(const std::string & } // regex: [^\r\n\p{L}\p{N}]?\p{L}+ //####FIXME: the first \p{L} is correct? - if (cpt != '\r' && cpt != '\n' && /*cpt_type != CODEPOINT_TYPE_LETTER &&*/ cpt_type != CODEPOINT_TYPE_NUMBER) { - if (cpt_type == CODEPOINT_TYPE_LETTER || _get_cpt_type(pos+1) == CODEPOINT_TYPE_LETTER) { // one or more letters + if (!(cpt == '\r' || cpt == '\n' || /*flags.is_letter |*/ flags.is_number)) { + if (flags.is_letter || _get_flags(pos+1).is_letter) { // one or more letters pos++; - while (_get_cpt_type(pos) == CODEPOINT_TYPE_LETTER) { + while (_get_flags(pos).is_letter) { pos++; } _add_token(pos); @@ -413,9 +405,9 @@ static std::vector unicode_regex_split_custom_llama3(const std::string & } // regex: \p{N}{1,3} - if (cpt_type == CODEPOINT_TYPE_NUMBER) { + if (flags.is_number) { size_t ini = pos; - while (_get_cpt_type(pos) == CODEPOINT_TYPE_NUMBER) { + while (_get_flags(pos).is_number) { if (++pos - ini >= 3 ) { _add_token(pos); ini = pos; @@ -426,14 +418,13 @@ static std::vector unicode_regex_split_custom_llama3(const std::string & } // regex: ?[^\s\p{L}\p{N}]+[\r\n]* - char32_t cpt2 = (cpt == ' ' ? _get_cpt(pos+1) : cpt); - int cpt2_type = (cpt == ' ' ? _get_cpt_type(pos+1) : cpt_type); - if (!unicode_cpt_is_whitespace(cpt2) && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_NUMBER && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) { + auto flags2 = (cpt == ' ' ? _get_flags(pos+1) : flags); + if (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number || flags2.is_undefined)) { pos += (cpt == ' '); - while (!unicode_cpt_is_whitespace(cpt2) && cpt2_type != CODEPOINT_TYPE_LETTER && cpt2_type != CODEPOINT_TYPE_NUMBER && cpt2_type != CODEPOINT_TYPE_UNIDENTIFIED) { - cpt2_type = _get_cpt_type(++pos); - cpt2 = _get_cpt(pos); + while (!(flags2.is_whitespace || flags2.is_letter || flags2.is_number || flags2.is_undefined)) { + flags2 = _get_flags(++pos); } + char32_t cpt2 = _get_cpt(pos); while (cpt2 == '\r' || cpt2 == '\n') { cpt2 = _get_cpt(++pos); } @@ -443,7 +434,7 @@ static std::vector unicode_regex_split_custom_llama3(const std::string & size_t num_whitespaces = 0; size_t last_end_r_or_n = 0; - while (unicode_cpt_is_whitespace(_get_cpt(pos+num_whitespaces))) { + while (_get_flags(pos+num_whitespaces).is_whitespace) { char32_t cpt2 = _get_cpt(pos+num_whitespaces); if (cpt2 == '\r' || cpt2 == '\n') { last_end_r_or_n = pos + num_whitespaces + 1; @@ -589,15 +580,14 @@ std::string unicode_cpt_to_utf8(uint32_t cp) { } std::vector unicode_cpts_normalize_nfd(const std::vector & cpts) { - std::vector result; - result.reserve(cpts.size()); + auto comp = [] (const uint32_t cpt, const range_nfd & range) { + return cpt < range.first; + }; + std::vector result(cpts.size()); for (size_t i = 0; i < cpts.size(); ++i) { - auto it = unicode_map_nfd.find(cpts[i]); - if (it == unicode_map_nfd.end()) { - result.push_back(cpts[i]); - } else { - result.push_back(it->second); - } + const uint32_t cpt = cpts[i]; + auto it = std::upper_bound(unicode_ranges_nfd.cbegin(), unicode_ranges_nfd.cend(), cpt, comp) - 1; + result[i] = (it->first <= cpt && cpt <= it->last) ? it->nfd : cpt; } return result; } @@ -611,31 +601,19 @@ std::vector unicode_cpts_from_utf8(const std::string & utf8) { return result; } -int unicode_cpt_type(uint32_t cp) { - static std::unordered_map cpt_types = unicode_cpt_type_map(); - const auto it = cpt_types.find(cp); - return it == cpt_types.end() ? CODEPOINT_TYPE_UNIDENTIFIED : it->second; +codepoint_flags unicode_cpt_flags(const uint32_t cp) { + static const codepoint_flags undef(codepoint_flags::UNDEFINED); + static const auto cpt_flags = unicode_cpt_flags_array(); + return cp < cpt_flags.size() ? cpt_flags[cp] : undef; } -int unicode_cpt_type(const std::string & utf8) { - if (utf8.length() == 0) { - return CODEPOINT_TYPE_UNIDENTIFIED; +codepoint_flags unicode_cpt_flags(const std::string & utf8) { + static const codepoint_flags undef(codepoint_flags::UNDEFINED); + if (utf8.empty()) { + return undef; // undefined } size_t offset = 0; - return unicode_cpt_type(unicode_cpt_from_utf8(utf8, offset)); -} - -bool unicode_cpt_is_whitespace(uint32_t cp) { - static const std::unordered_set is_whitespace = [] { - std::unordered_set is_whitespace; - for (auto p : unicode_ranges_whitespace) { - for (auto i = p.first; i <= p.second; ++i) { - is_whitespace.insert(i); - } - } - return is_whitespace; - }(); - return (bool)is_whitespace.count(cp); + return unicode_cpt_flags(unicode_cpt_from_utf8(utf8, offset)); } std::string unicode_byte_to_utf8(uint8_t byte) { @@ -656,21 +634,21 @@ char32_t unicode_tolower(char32_t cp) { std::vector unicode_regex_split(const std::string & text, const std::vector & regex_exprs) { // unicode categories static const std::map k_ucat_enum = { - { "\\p{N}", CODEPOINT_TYPE_NUMBER }, - { "\\p{L}", CODEPOINT_TYPE_LETTER }, - { "\\p{P}", CODEPOINT_TYPE_PUNCTUATION }, + { "\\p{N}", codepoint_flags::NUMBER }, + { "\\p{L}", codepoint_flags::LETTER }, + { "\\p{P}", codepoint_flags::PUNCTUATION }, }; static const std::map k_ucat_cpt = { - { CODEPOINT_TYPE_NUMBER, 0xD1 }, - { CODEPOINT_TYPE_LETTER, 0xD2 }, - { CODEPOINT_TYPE_PUNCTUATION, 0xD3 }, + { codepoint_flags::NUMBER, 0xD1 }, + { codepoint_flags::LETTER, 0xD2 }, + { codepoint_flags::PUNCTUATION, 0xD3 }, }; static const std::map k_ucat_map = { - { CODEPOINT_TYPE_NUMBER, "\x30-\x39" }, // 0-9 - { CODEPOINT_TYPE_LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z - { CODEPOINT_TYPE_PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\} + { codepoint_flags::NUMBER, "\x30-\x39" }, // 0-9 + { codepoint_flags::LETTER, "\x41-\x5A\x61-\x7A" }, // A-Za-z + { codepoint_flags::PUNCTUATION, "\x21-\x23\x25-\x2A\x2C-\x2F\x3A-\x3B\x3F-\x40\\\x5B-\\\x5D\x5F\\\x7B\\\x7D" }, // !-#%-*,-/:-;?-@\[-\]_\{\} }; // compute collapsed codepoints only if needed by at least one regex @@ -701,10 +679,10 @@ std::vector unicode_regex_split(const std::string & text, const std continue; } - const int cpt_type = unicode_cpt_type(cpts[i]); + const int cpt_flag = unicode_cpt_flags(cpts[i]).category_flag(); - if (k_ucat_cpt.find(cpt_type) != k_ucat_cpt.end()) { - text_collapsed[i] = k_ucat_cpt.at(cpt_type); + if (k_ucat_cpt.find(cpt_flag) != k_ucat_cpt.end()) { + text_collapsed[i] = k_ucat_cpt.at(cpt_flag); } else { text_collapsed[i] = (char) 0xD0; // fallback } diff --git a/unicode.h b/unicode.h index d6a14d470..7513be4ad 100644 --- a/unicode.h +++ b/unicode.h @@ -4,24 +4,56 @@ #include #include -#define CODEPOINT_TYPE_UNIDENTIFIED 0 -#define CODEPOINT_TYPE_NUMBER 1 -#define CODEPOINT_TYPE_LETTER 2 -#define CODEPOINT_TYPE_SEPARATOR 3 -#define CODEPOINT_TYPE_ACCENT_MARK 4 -#define CODEPOINT_TYPE_PUNCTUATION 5 -#define CODEPOINT_TYPE_SYMBOL 6 -#define CODEPOINT_TYPE_CONTROL 7 +struct codepoint_flags { + enum { + UNDEFINED = 0x0001, + NUMBER = 0x0002, // regex: \p{N} + LETTER = 0x0004, // regex: \p{L} + SEPARATOR = 0x0008, // regex: \p{Z} + ACCENT_MARK = 0x0010, // regex: \p{M} + PUNCTUATION = 0x0020, // regex: \p{P} + SYMBOL = 0x0040, // regex: \p{S} + CONTROL = 0x0080, // regex: \p{C} + MASK_CATEGORIES = 0x00FF, + }; + + // codepoint type + uint16_t is_undefined : 1; + uint16_t is_number : 1; // regex: \p{N} + uint16_t is_letter : 1; // regex: \p{L} + uint16_t is_separator : 1; // regex: \p{Z} + uint16_t is_accent_mark : 1; // regex: \p{M} + uint16_t is_punctuation : 1; // regex: \p{P} + uint16_t is_symbol : 1; // regex: \p{S} + uint16_t is_control : 1; // regex: \p{C} + // helper flags + uint16_t is_whitespace : 1; // regex: \s + uint16_t is_lowercase : 1; + uint16_t is_uppercase : 1; + uint16_t is_nfd : 1; + + // decode from uint16 + inline codepoint_flags(const uint16_t flags=0) { + *reinterpret_cast(this) = flags; + } + + inline uint16_t as_uint() const { + return *reinterpret_cast(this); + } + + inline uint16_t category_flag() const { + return this->as_uint() & MASK_CATEGORIES; + } +}; + std::string unicode_cpt_to_utf8(uint32_t cp); std::vector unicode_cpts_from_utf8(const std::string & utf8); std::vector unicode_cpts_normalize_nfd(const std::vector & cpts); -int unicode_cpt_type(uint32_t cp); -int unicode_cpt_type(const std::string & utf8); - -bool unicode_cpt_is_whitespace(uint32_t cp); +codepoint_flags unicode_cpt_flags(const uint32_t cp); +codepoint_flags unicode_cpt_flags(const std::string & utf8); std::string unicode_byte_to_utf8(uint8_t byte); uint8_t unicode_utf8_to_byte(const std::string & utf8); From ef277de2add255a08b2b909ebfbf70364d1f4dc4 Mon Sep 17 00:00:00 2001 From: Engininja2 <139037756+Engininja2@users.noreply.github.com> Date: Fri, 17 May 2024 18:39:25 -0600 Subject: [PATCH 056/137] cmake : fix typo in AMDGPU_TARGETS (#7356) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 990e34b86..5e4daae03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -581,7 +581,7 @@ if (LLAMA_HIPBLAS) else() # Forward AMDGPU_TARGETS to CMAKE_HIP_ARCHITECTURES. if(AMDGPU_TARGETS AND NOT CMAKE_HIP_ARCHITECTURES) - set(CMAKE_HIP_ARCHITECTURES ${AMDGPU_ARGETS}) + set(CMAKE_HIP_ARCHITECTURES ${AMDGPU_TARGETS}) endif() cmake_minimum_required(VERSION 3.21) enable_language(HIP) From 05834841dcb4f922983ea976539c70472272df9a Mon Sep 17 00:00:00 2001 From: slaren Date: Sat, 18 May 2024 02:39:54 +0200 Subject: [PATCH 057/137] ggml : fix quants nans when all the group weights are very close to zero (#7313) --- ggml-quants.c | 30 ++++++++++++++++++------------ tests/test-backend-ops.cpp | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/ggml-quants.c b/ggml-quants.c index 9b291d522..7008e5d8b 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -14,6 +14,12 @@ #include // for qsort #include // for GGML_ASSERT +#define GROUP_MAX_EPS 1e-15f +#define GROUP_MAX_EPS_IQ3_XXS 1e-8f +#define GROUP_MAX_EPS_IQ2_S 1e-8f +#define GROUP_MAX_EPS_IQ1_M 1e-7f +#define GROUP_MAX_EPS_IQ1_S 1e-12f + #if defined(_MSC_VER) // disable "possible loss of data" to avoid warnings for hundreds of casts // we should just be careful :) @@ -1109,7 +1115,7 @@ static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * float ax = fabsf(x[i]); if (ax > amax) { amax = ax; max = x[i]; } } - if (amax < 1e-30f) { // all zero + if (amax < GROUP_MAX_EPS) { // all zero for (int i = 0; i < n; ++i) { L[i] = 0; } @@ -1177,7 +1183,7 @@ static float make_q3_quants(int n, int nmax, const float * restrict x, int8_t * float ax = fabsf(x[i]); if (ax > amax) { amax = ax; max = x[i]; } } - if (!amax) { // all zero + if (amax < GROUP_MAX_EPS) { // all zero for (int i = 0; i < n; ++i) { L[i] = 0; } return 0.f; } @@ -1646,7 +1652,7 @@ static float make_qp_quants(int n, int nmax, const float * restrict x, uint8_t * break; } } - return sumlx / suml2; + return sumlx/suml2; } static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restrict y, int k, const float * restrict quant_weights) { @@ -2653,7 +2659,7 @@ void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict } - if (!max_abs_scale) { + if (max_abs_scale < GROUP_MAX_EPS) { memset(&y[i], 0, sizeof(block_q6_K)); y[i].d = GGML_FP32_TO_FP16(0.f); x += QK_K; @@ -2805,7 +2811,7 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri } - if (!max_abs_scale) { + if (max_abs_scale < GROUP_MAX_EPS) { memset(&y[i], 0, sizeof(block_q6_K)); y[i].d = GGML_FP32_TO_FP16(0.f); x += QK_K; @@ -12599,7 +12605,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict } float max = xval[0]; for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]); - if (!max) { + if (max < GROUP_MAX_EPS) { scales[ib] = 0; memset(L, 0, 32); continue; @@ -12775,7 +12781,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v } float max = xval[0]; for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]); - if (!max) { + if (max < GROUP_MAX_EPS) { scales[ib] = 0; memset(L, 0, 16); continue; @@ -13216,7 +13222,7 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, v } float max = xval[0]; for (int i = 1; i < 32; ++i) max = MAX(max, xval[i]); - if (!max) { + if (max < GROUP_MAX_EPS_IQ3_XXS) { scales[ib] = 0; memset(L, 0, 32); continue; @@ -13756,7 +13762,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); float max = fabsf(xb[0]); for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i])); - if (!max) { + if (max < GROUP_MAX_EPS_IQ1_S) { scales[ib] = 0; memset(L, 1, block_size); continue; @@ -13944,7 +13950,7 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy } float max = fabsf(xb[0]); for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i])); - if (!max) { + if (max < GROUP_MAX_EPS_IQ1_M) { scales[ib] = 0; memset(L, 1, block_size); continue; @@ -14208,7 +14214,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block amax = ax; max = xb[j]; } } - if (!amax) { + if (amax < GROUP_MAX_EPS) { scales[ib] = 0; continue; } @@ -14429,7 +14435,7 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy } float max = xval[0]; for (int i = 1; i < 16; ++i) max = MAX(max, xval[i]); - if (!max) { + if (max < GROUP_MAX_EPS_IQ2_S) { scales[ib] = 0; continue; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 85ef21c2a..c74e253db 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -16,6 +16,7 @@ #include #include + static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { // static RNG initialization (revisit if n_threads stops being constant) static const size_t n_threads = std::thread::hardware_concurrency(); @@ -49,6 +50,22 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m t.join(); } +#if 0 + const char * val_str = getenv("GGML_TEST_EPS"); + float val = 1e-9f; + if (val_str != nullptr) { + val = std::stof(val_str); + printf("GGML_TEST_EPS=%e\n", val); + } + + // test quantization with very small values that may result in nan scales due to division by zero + if (ggml_is_quantized(tensor->type)) { + for (int i = 0; i < 256; i++) { + data[i] = val; + } + } +#endif + if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) { ggml_backend_tensor_set(tensor, data.data(), 0, size * sizeof(float)); } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { @@ -64,6 +81,7 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m } } ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, size/tensor->ne[0], tensor->ne[0], im); + GGML_ASSERT(ggml_validate_row_data(tensor->type, dataq.data(), dataq.size())); ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || tensor->type == GGML_TYPE_I32) { // This is going to create some weird integers though. From b49a13dd2fa9c94c2c19a8c248bb7fa45499f9a8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 18 May 2024 08:46:20 +0300 Subject: [PATCH 058/137] convert : fix set_vocab_sentencepiece (#6866) * convert : fix set_vocab_sentencepiece * Update convert-hf-to-gguf.py --- convert-hf-to-gguf.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 5ba3161c7..cd1750aa3 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -573,6 +573,10 @@ class Model: vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size()) + tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)] + scores: list[float] = [-10000.0] * vocab_size + toktypes: list[int] = [SentencePieceTokenTypes.UNKNOWN] * vocab_size + for token_id in range(tokenizer.vocab_size()): piece = tokenizer.IdToPiece(token_id) text = piece.encode("utf-8") @@ -588,21 +592,23 @@ class Model: elif tokenizer.IsByte(token_id): toktype = SentencePieceTokenTypes.BYTE - tokens.append(text) - scores.append(score) - toktypes.append(toktype) + tokens[token_id] = text + scores[token_id] = score + toktypes[token_id] = toktype added_tokens_file = self.dir_model / 'added_tokens.json' if added_tokens_file.is_file(): with open(added_tokens_file, "r", encoding="utf-8") as f: added_tokens_json = json.load(f) - for key in added_tokens_json: - key = key.encode("utf-8") - if key not in tokens: - tokens.append(key) - scores.append(-1000.0) - toktypes.append(SentencePieceTokenTypes.USER_DEFINED) + token_id = added_tokens_json[key] + if (token_id >= vocab_size): + logger.warning(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}') + continue + + tokens[token_id] = key.encode("utf-8") + scores[token_id] = -1000.0 + toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED if vocab_size > len(tokens): pad_count = vocab_size - len(tokens) @@ -612,8 +618,6 @@ class Model: scores.append(-1000.0) toktypes.append(SentencePieceTokenTypes.UNUSED) - assert len(tokens) == vocab_size - self.gguf_writer.add_tokenizer_model("llama") self.gguf_writer.add_tokenizer_pre("default") self.gguf_writer.add_token_list(tokens) From de731963441ff128248259e1b99573d75264d210 Mon Sep 17 00:00:00 2001 From: Brian Date: Sat, 18 May 2024 16:04:23 +1000 Subject: [PATCH 059/137] github-actions-labeler: initial commit (#7330) * github-actions-labeler: initial commit [no ci] * github actions: remove priority auto labeling [no ci] --- .github/labeler.yml | 73 +++++++++++++++++++++++++++++++++++ .github/workflows/labeler.yml | 12 ++++++ 2 files changed, 85 insertions(+) create mode 100644 .github/labeler.yml create mode 100644 .github/workflows/labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000..fca60594f --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,73 @@ +# https://github.com/actions/labeler + +SYCL: + - changed-files: + - any-glob-to-any-file: + - ggml-sycl.h + - ggml-sycl.cpp + - README-sycl.md +Nvidia GPU: + - changed-files: + - any-glob-to-any-file: + - ggml-cuda/** +Vulkan: + - changed-files: + - any-glob-to-any-file: + - ggml_vk_generate_shaders.py + - ggml-vulkan* +documentation: + - changed-files: + - any-glob-to-any-file: + - docs/** + - media/** +testing: + - changed-files: + - any-glob-to-any-file: + - tests/** +build: + - changed-files: + - any-glob-to-any-file: + - cmake/** + - CMakeLists.txt + - CMakePresets.json + - codecov.yml +examples: + - changed-files: + - any-glob-to-any-file: examples/** +devops: + - changed-files: + - any-glob-to-any-file: + - .devops/** + - .github/** + - ci/** +python: + - changed-files: + - any-glob-to-any-file: + - "**/*.py" + - requirements/** + - gguf-py/** + - .flake8 +script: + - changed-files: + - any-glob-to-any-file: + - scripts/** +android: + - changed-files: + - any-glob-to-any-file: + - examples/llama.android/** +server: + - changed-files: + - any-glob-to-any-file: + - examples/server/** +ggml: + - changed-files: + - any-glob-to-any-file: + - ggml-*.c + - ggml-*.h + - ggml-cuda/** +nix: + - changed-files: + - any-glob-to-any-file: + - "**/*.nix" + - .github/workflows/nix-*.yml + - .devops/nix/nixpkgs-instances.nix diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 000000000..e57cd86e2 --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,12 @@ +name: "Pull Request Labeler" +on: +- pull_request_target + +jobs: + labeler: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v5 From c1b295eea5c49887a066559527a74e8b94fe9db0 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Sat, 18 May 2024 08:10:58 +0200 Subject: [PATCH 060/137] Update and fix Vulkan soft_max and argsort implementations (#7237) * Update and fix Vulkan softmax implementation * Update and fix Vulkan argsort implementation --- ggml-vulkan-shaders.hpp | 2157 ++++++++++++++++++----------------- ggml-vulkan.cpp | 194 ++-- ggml_vk_generate_shaders.py | 86 +- 3 files changed, 1210 insertions(+), 1227 deletions(-) diff --git a/ggml-vulkan-shaders.hpp b/ggml-vulkan-shaders.hpp index d3a2fcf38..586bafe46 100644 --- a/ggml-vulkan-shaders.hpp +++ b/ggml-vulkan-shaders.hpp @@ -363,320 +363,349 @@ const uint64_t add_f32_len = 4276; unsigned char argsort_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x2b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x41,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0b,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x04,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x10,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x04,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x30,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x30,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x89,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x8a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8c,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0f,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x10,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x11,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x2b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x31,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x31,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x31,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x81,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x82,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x82,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x84,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x84,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x12,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x12,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x1c,0x01,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, 0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x0d,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x10,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x11,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x12,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x22,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x26,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x23,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1e,0x00,0x04,0x00, -0x31,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x32,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x31,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x32,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x00,0x04,0x00,0x00,0x1c,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x12,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x21,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x22,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x22,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x25,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x22,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1e,0x00,0x05,0x00,0x30,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x31,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x31,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, 0x34,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, 0x14,0x00,0x02,0x00,0x37,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x80,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x83,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x8e,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x00,0x04,0x00,0x00, -0x2c,0x00,0x06,0x00,0x22,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x2b,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x88,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x8b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x8b,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x93,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x0f,0x01,0x00,0x00,0x0d,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x11,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x2c,0x00,0x06,0x00,0x21,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x07,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x25,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x27,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x26,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x34,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x1d,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x24,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x25,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x25,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x34,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0x36,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, 0x36,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x3a,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x38,0x00,0x00,0x00, 0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x37,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x36,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x4c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x4a,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x4b,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x4f,0x00,0x00,0x00,0x29,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x4c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x4c,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x50,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x53,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x53,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x50,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x56,0x00,0x00,0x00,0xb2,0x00,0x05,0x00,0x37,0x00,0x00,0x00, -0x5b,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x36,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x54,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x50,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5f,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0x5e,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x62,0x00,0x00,0x00,0xac,0x00,0x05,0x00, -0x37,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0x25,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x61,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x65,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x60,0x00,0x00,0x00,0xc6,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0x27,0x01,0x00,0x00,0xac,0x00,0x05,0x00,0x37,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x30,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x70,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x70,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x00,0x00,0x00, -0xc7,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xaa,0x00,0x05,0x00, -0x37,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x77,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x75,0x00,0x00,0x00, -0x76,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x76,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0xab,0x00,0x05,0x00,0x37,0x00,0x00,0x00, -0x7b,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7b,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x16,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00, -0x99,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0xba,0x00,0x05,0x00, -0x37,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7f,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x8e,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x80,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x16,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0xad,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0xb8,0x00,0x05,0x00,0x37,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7f,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x37,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x9b,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x39,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x34,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x44,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x45,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x46,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x48,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x48,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x45,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x4b,0x00,0x00,0x00, +0xb2,0x00,0x05,0x00,0x37,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x36,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x4a,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x49,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x49,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x45,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x54,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x54,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x53,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x57,0x00,0x00,0x00,0xac,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x24,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x56,0x00,0x00,0x00,0x57,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x55,0x00,0x00,0x00,0xc6,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0xac,0x00,0x05,0x00,0x37,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x63,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x64,0x00,0x00,0x00,0xc7,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0xaa,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0x6a,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x6c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6a,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6b,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0xa8,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x76,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x74,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x76,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x75,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x13,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x37,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7d,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x34,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0xaa,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x87,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x83,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x44,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x93,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x88,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x93,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x88,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0xba,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x87,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x93,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x88,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x93,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x88,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xb8,0x00,0x05,0x00,0x37,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x87,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x37,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x9e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7f,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x37,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x87,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x37,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, 0xb5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2a,0x01,0x00,0x00,0xb4,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x0f,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00, -0x13,0x01,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0f,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x13,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x15,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x5f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x25,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x29,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, 0xb5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb5,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x77,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc0,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x34,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0xab,0x00,0x05,0x00,0x37,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xc6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xc5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x16,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcf,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0xd8,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xb8,0x00,0x05,0x00, -0x37,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00, -0xe3,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x8e,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xe6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x80,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x16,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x13,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xef,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x8e,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x80,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0xba,0x00,0x05,0x00,0x37,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x37,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0xdc,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0xf3,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x29,0x01,0x00,0x00,0xf5,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, -0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0x1c,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x16,0x00,0x00,0x00, -0x20,0x01,0x00,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x1c,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x20,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf6,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x77,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x70,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x70,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x50,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x62,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x62,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x50,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5f,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x61,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x56,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x56,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x26,0x01,0x00,0x00, -0x50,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x53,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x55,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x01,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0xf9,0x00,0x02,0x00,0x6c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbc,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x5f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x37,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0xa8,0x00,0x04,0x00,0x37,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xc6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xcf,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xce,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x34,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xaa,0x00,0x05,0x00,0x37,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xd5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd2,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0x44,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x93,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x88,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xdc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0xe2,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x93,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x88,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, +0xb8,0x00,0x05,0x00,0x37,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe7,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x44,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x93,0x00,0x00,0x00, +0xee,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x88,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0xf3,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x93,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x88,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0xf6,0x00,0x00,0x00,0xba,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x37,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0xe6,0x00,0x00,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xcf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcf,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x37,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xc6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x37,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xfd,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xfc,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x00,0x00,0x00,0x32,0x01,0x00,0x00, +0x13,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0xbe,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x32,0x01,0x00,0x00, +0x37,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xbe,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xfd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xfd,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x6c,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x6c,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x65,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x65,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x45,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x46,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x57,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x57,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0x45,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x54,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x56,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x45,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x48,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4a,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x37,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x2f,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x0e,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x0c,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x0e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x0d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x41,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x44,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x1a,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x3e,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1b,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x0e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x0e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x1d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1d,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t argsort_f32_len = 3752; +const uint64_t argsort_f32_len = 4096; unsigned char clamp_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -76253,430 +76282,15 @@ const uint64_t silu_f32_len = 1264; unsigned char soft_max_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x5c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, 0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, 0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0d,0x00,0x05,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00, 0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x10,0x00,0x06,0x00, -0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x72,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x89,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8a,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x8a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8c,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa0,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xa1,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xa1,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x06,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x07,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x07,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x09,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x09,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x0a,0x00, -0x17,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x18,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x1c,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x14,0x00,0x02,0x00, -0x27,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x1c,0x00,0x04,0x00,0x5a,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x5b,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x5b,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x60,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x72,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x74,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x7c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x89,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x8a,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x8b,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x8a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x8b,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xa0,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xa1,0x00,0x00,0x00, -0xa0,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa2,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xa2,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x08,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0xb2,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, -0x06,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x08,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x07,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x45,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0x00,0x00,0x80,0xff,0x36,0x00,0x05,0x00, -0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x26,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0xba,0x00,0x05,0x00, -0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x28,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x3b,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x48,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x48,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x4c,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x48,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x48,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00, -0x4b,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, -0x16,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x2a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x5b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x64,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x85,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x95,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x94,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x9a,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x9e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x9b,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9d,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0x7c,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0xa6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9e,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0xa7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x81,0x00,0x05,0x00, -0x16,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0xab,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x61,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x67,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00, -0x48,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x64,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x1a,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xb2,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xad,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xb5,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x49,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xbf,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xbd,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xbe,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, -0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0xca,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbf,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xbf,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb6,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0x1b,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb5,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x60,0x00,0x00,0x00, -0xcf,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0xcf,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x61,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x0f,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xd8,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x6c,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd7,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdd,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x7c,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xe9,0x00,0x00,0x00, -0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0xee,0x00,0x00,0x00, -0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xf1,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xee,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf0,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0xf9,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x06,0x00, -0x16,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x61,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x61,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x7c,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x09,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xe4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x0c,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd8,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd8,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0x4a,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x10,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x10,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x1a,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0xb2,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0xad,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x16,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x16,0x01,0x00,0x00,0x11,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x11,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0x4b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x01,0x00,0x00, -0xf7,0x00,0x03,0x00,0x1c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x1a,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x1c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1b,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0x0f,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x5c,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x61,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x25,0x01,0x00,0x00,0x23,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x61,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x1c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x1c,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x13,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x13,0x01,0x00,0x00,0xc3,0x00,0x05,0x00, -0x1a,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x10,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x12,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0xcf,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x4c,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0x27,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x31,0x01,0x00,0x00, -0x30,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x37,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x30,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x13,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, -0x09,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x16,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x3f,0x01,0x00,0x00,0x41,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x2f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x31,0x01,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t soft_max_f32_len = 4916; - -unsigned char soft_max_f32_f16_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x61,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, -0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, -0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, -0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0d,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, -0x0e,0x01,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x01,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, 0x11,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, 0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, @@ -76692,55 +76306,45 @@ unsigned char soft_max_f32_f16_data[] = { 0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, 0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, 0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x1c,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x17,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x72,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x73,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x74,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x75,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x8c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x8d,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x8d,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x8d,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x8f,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8f,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x09,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x73,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x75,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x75,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8a,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x8b,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x8d,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x8d,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa3,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xa4,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xa4,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa6,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x21,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x0c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x0c,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0e,0x01,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x0e,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x11,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x16,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x1e,0x00,0x0a,0x00,0x17,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x0a,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0c,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0c,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x16,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x1e,0x00,0x09,0x00, +0x17,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, 0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, 0x16,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, 0x18,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x17,0x00,0x00,0x00, @@ -76750,56 +76354,49 @@ unsigned char soft_max_f32_f16_data[] = { 0x1a,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x1c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x80,0x3f,0x2b,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00, 0x20,0x00,0x04,0x00,0x24,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x16,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x27,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0x1c,0x00,0x04,0x00,0x5a,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x59,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5b,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x5b,0x00,0x00,0x00,0x5c,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x60,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0x28,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x74,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x75,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x76,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x7e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x72,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x73,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x74,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x74,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x7c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x1a,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x89,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x8a,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x8b,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x8c,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x94,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x89,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xa3,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xa4,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa5,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xa5,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x8c,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x8d,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x8e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x8e,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x1c,0x00,0x04,0x00, +0xa0,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa1,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xa5,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, 0x08,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, -0x0b,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x0d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0d,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x09,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x0b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, 0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x4a,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x00,0x00,0x80,0xff,0x36,0x00,0x05,0x00, +0x52,0x01,0x00,0x00,0x9d,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x00,0x00,0x80,0xff,0x36,0x00,0x05,0x00, 0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, 0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, 0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, @@ -76816,275 +76413,693 @@ unsigned char soft_max_f32_f16_data[] = { 0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, 0x26,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0xba,0x00,0x05,0x00, -0x27,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x28,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x2f,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x37,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x3b,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x38,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x38,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x38,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x3b,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x48,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x35,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x4c,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x47,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x4a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x48,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x4c,0x00,0x00,0x00,0x82,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x34,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x51,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x52,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x48,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x48,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x4b,0x00,0x00,0x00,0x47,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x4c,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x57,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, -0x16,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x57,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x2a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x59,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x58,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x64,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x0f,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x67,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x4d,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x66,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x65,0x00,0x00,0x00, -0x66,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7b,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x7c,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0x7b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x29,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x39,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x36,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x39,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x39,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x39,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x3c,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x49,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x36,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x48,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x49,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4d,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x49,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x49,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x4c,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, +0x16,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x58,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2b,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x56,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x2b,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x1c,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x67,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x71,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x70,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x60,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x71,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x7e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x87,0x00,0x00,0x00, 0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x88,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x85,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x94,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x8d,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x89,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x98,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x88,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x88,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x97,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x16,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x81,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x9c,0x00,0x00,0x00,0xac,0x00,0x05,0x00,0x27,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x9e,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa0,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x94,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x89,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x16,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa1,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xac,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xa1,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0xab,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0xae,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x67,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x67,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, -0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x66,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x1a,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0xb6,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xad,0x00,0x05,0x00, -0x27,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, -0xba,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xbd,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb8,0x00,0x00,0x00,0x7c,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0xc3,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x5c,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, -0x16,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0xcd,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00,0xce,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc3,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xba,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xba,0x00,0x00,0x00, -0xc3,0x00,0x05,0x00,0x1a,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0x4e,0x01,0x00,0x00,0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb9,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x60,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0x5c,0x00,0x00,0x00,0x6a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd9,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x0f,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0xdc,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x4f,0x01,0x00,0x00,0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdb,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe1,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xda,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x13,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, -0x4f,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00, -0xeb,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x6a,0x00,0x00,0x00, -0xe8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x24,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x7f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xac,0x00,0x05,0x00, -0x27,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf2,0x00,0x00,0x00, -0xf4,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x4f,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x94,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, -0x6a,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x89,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0xfd,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xff,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x53,0x01,0x00,0x00, -0xfe,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, -0x01,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x53,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0x01,0x01,0x00,0x00,0xd4,0x00,0x00,0x00,0x0c,0x00,0x06,0x00, -0x16,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x61,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x09,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x61,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x7c,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0x6a,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x11,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x4f,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdb,0x00,0x00,0x00, -0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x15,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x15,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x1a,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0xb6,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x18,0x01,0x00,0x00, -0xad,0x00,0x05,0x00,0x27,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x17,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x1b,0x01,0x00,0x00,0x16,0x01,0x00,0x00, -0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x16,0x01,0x00,0x00, -0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x27,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0xf7,0x00,0x03,0x00,0x21,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x1f,0x01,0x00,0x00,0x20,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x20,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00, -0x0f,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x60,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x5c,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, -0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x2a,0x01,0x00,0x00,0x61,0x00,0x00,0x00, -0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x61,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x21,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x21,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x18,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x18,0x01,0x00,0x00,0xc3,0x00,0x05,0x00, -0x1a,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x15,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x17,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0x16,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0xd3,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x87,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x7e,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x97,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x99,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8a,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x16,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x98,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, +0x16,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x9b,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x61,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x60,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xa5,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa6,0x00,0x00,0x00,0x56,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xab,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x1a,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0xaa,0x00,0x00,0x00, +0x60,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xae,0x00,0x00,0x00, +0xad,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xad,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb1,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xac,0x00,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xb7,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb5,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb6,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xa5,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa6,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb7,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xae,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xae,0x00,0x00,0x00,0xc3,0x00,0x05,0x00,0x1a,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x1b,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xab,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xa5,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa6,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x11,0x01,0x00,0x00, +0xce,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0xd3,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x66,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd3,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x0f,0x00,0x00,0x00, +0xae,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xdd,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xcd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x7e,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0xe5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00,0xac,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf2,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xef,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x7e,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0xfb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xfd,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0xfc,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x16,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x0c,0x00,0x06,0x00,0x16,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0xa6,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa6,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x7e,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x0f,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xce,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xce,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xcb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xcd,0x00,0x00,0x00,0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x13,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x13,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x1a,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0xaa,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0xad,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x19,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x14,0x01,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x0f,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x1f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1d,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xa5,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0xa2,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0xa6,0x00,0x00,0x00,0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa6,0x00,0x00,0x00,0x29,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1f,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x16,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x16,0x01,0x00,0x00, +0xc3,0x00,0x05,0x00,0x1a,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x13,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x15,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0xc6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x31,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x31,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x34,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x66,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x33,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x39,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x32,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0xae,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x43,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x41,0x01,0x00,0x00, +0x42,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x42,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x33,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x43,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x13,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4a,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x7e,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x16,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x4c,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, 0xf9,0x00,0x02,0x00,0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x34,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x17,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0x27,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x6c,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x36,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x3c,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x36,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x13,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x7c,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x0e,0x01,0x00,0x00,0x6a,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x16,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x31,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x44,0x01,0x00,0x00,0x46,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x59,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x34,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x36,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x31,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x33,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t soft_max_f32_len = 4836; + +unsigned char soft_max_f32_f16_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x73,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0c,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x11,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x17,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x74,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x75,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x75,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x77,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x77,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x8d,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x8e,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x90,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x90,0x00,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0d,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x0e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x0e,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x10,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x10,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x56,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x16,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x1e,0x00,0x09,0x00,0x17,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x18,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x18,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x1c,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x80,0x3f,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x24,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x28,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x1a,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x74,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x76,0x00,0x00,0x00, +0x77,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x7e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x8c,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x8d,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x8e,0x00,0x00,0x00, +0x8d,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x8f,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8f,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa0,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x1c,0x00,0x04,0x00, +0xa3,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0xa0,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xa4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xa4,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x08,0x01,0x00,0x00,0x2b,0x00,0x04,0x00,0x1a,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x1d,0x00,0x03,0x00, +0x0d,0x01,0x00,0x00,0x16,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x0f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0f,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2c,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0xa0,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x00,0x00,0x80,0xff,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x1d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0xba,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x27,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x2b,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x29,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x1e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x1c,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x39,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x36,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x3d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x24,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x39,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x39,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x39,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x3c,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3d,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x49,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x36,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x48,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x49,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4d,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x52,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x49,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x49,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0x4c,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x58,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, +0x16,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x58,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2b,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2b,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x49,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5e,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x72,0x01,0x00,0x00,0x2b,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x59,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x1c,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0x67,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x60,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x67,0x00,0x00,0x00,0x5f,0x00,0x00,0x00,0x60,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x71,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6f,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x70,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x60,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x71,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x7b,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x7e,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x8a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x87,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x89,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x96,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x9a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x8a,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9c,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8a,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x8a,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x9b,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x9e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x61,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x61,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0xa0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x5e,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x60,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xa8,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa9,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x1a,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0xad,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0xad,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x64,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb0,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaf,0x00,0x00,0x00,0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xba,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xa8,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x16,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa9,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xba,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xba,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xc3,0x00,0x05,0x00, +0x1a,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xae,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb0,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xa8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xe0,0x00,0x04,0x00, +0x4e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0xa9,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xce,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xce,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0xd1,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xcf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x61,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xe0,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xe0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe0,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x41,0x00,0x06,0x00, +0x7e,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x24,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xac,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0x1e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xf5,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x66,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfc,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x97,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x8c,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x73,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0xff,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x85,0x00,0x05,0x00, +0x16,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x01,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x16,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0xf4,0x00,0x00,0x00,0x27,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x16,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0xca,0x00,0x00,0x00,0x0c,0x00,0x06,0x00, +0x16,0x00,0x00,0x00,0x06,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0xa9,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa9,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x7e,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x13,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd1,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0xa0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xce,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd0,0x00,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x17,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x1a,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0xad,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0xad,0x00,0x05,0x00,0x28,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x64,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x19,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x1d,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x18,0x01,0x00,0x00, +0x7c,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x0f,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x21,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xa8,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0xa5,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x16,0x00,0x00,0x00, +0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0xa9,0x00,0x00,0x00, +0x81,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa9,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x23,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x4e,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x1a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1a,0x01,0x00,0x00,0xc3,0x00,0x05,0x00, +0x1a,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x69,0x01,0x00,0x00, +0x1b,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0xc9,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x35,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0x28,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x66,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x37,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x3d,0x01,0x00,0x00,0x36,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x0f,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x28,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x66,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x47,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x46,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x46,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x47,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4c,0x01,0x00,0x00,0x13,0x00,0x00,0x00,0x66,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0x7e,0x00,0x00,0x00,0x50,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x64,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x16,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x88,0x00,0x05,0x00,0x16,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x50,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x38,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x38,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0xa0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x35,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t soft_max_f32_f16_len = 5000; +const uint64_t soft_max_f32_f16_len = 4904; unsigned char split_k_reduce_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index b9449be03..d1ba47ac3 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -294,7 +294,6 @@ struct vk_op_rope_neox_push_constants { struct vk_op_soft_max_push_constants { uint32_t KX; uint32_t KY; - uint32_t KZ; float scale; float max_bias; float m0; @@ -304,7 +303,8 @@ struct vk_op_soft_max_push_constants { struct vk_op_argsort_push_constants { uint32_t ncols; - bool ascending; + uint32_t ncols_pad; + int32_t order; }; // Allow pre-recording command buffers @@ -1501,8 +1501,8 @@ static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { ggml_vk_create_pipeline(ctx, ctx->device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {512, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->device->pipeline_soft_max_f32_f16, "soft_max_f32_f16", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 4, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_soft_max_f32, "soft_max_f32", soft_max_f32_len, soft_max_f32_data, "main", 3, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_soft_max_f32_f16, "soft_max_f32_f16", soft_max_f32_f16_len, soft_max_f32_f16_data, "main", 3, sizeof(vk_op_soft_max_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_f32, "rope_f32", rope_f32_len, rope_f32_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_f16, "rope_f16", rope_f16_len, rope_f16_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); @@ -3752,7 +3752,7 @@ static void ggml_vk_op_repeat(ggml_backend_vk_context * ctx, vk_context * subctx } -static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, ggml_op op) { +static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_ADD: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { @@ -3834,7 +3834,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const if (src0->type == GGML_TYPE_F32 && (src1 == nullptr || src1->type == GGML_TYPE_F32) && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_soft_max_f32; } - if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16 && src2->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) { + if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_soft_max_f32_f16; } return nullptr; @@ -3900,15 +3900,12 @@ static bool ggml_vk_op_supports_incontiguous(ggml_op op) { } template -static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, ggml_op op, const PC&& pc) { +static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op, const PC&& pc) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; if (src1 != nullptr) { std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; } - if (src2 != nullptr) { - std::cerr << "), (" << src2 << ", name=" << src2->name << ", type=" << src2->type << ", backend=" << src2->backend << ", ne0=" << src2->ne[0] << ", ne1=" << src2->ne[1] << ", ne2=" << src2->ne[2] << ", ne3=" << src2->ne[3] << ", nb0=" << src2->nb[0] << ", nb1=" << src2->nb[1] << ", nb2=" << src2->nb[2] << ", nb3=" << src2->nb[3]; - } std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "), " << ggml_op_name(op) << ")" << std::endl; #endif GGML_ASSERT(op == GGML_OP_GET_ROWS || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT @@ -3929,10 +3926,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c const uint64_t nb2 = dst->nb[2]; const uint64_t nb3 = dst->nb[3]; - const bool use_src2 = src2 != nullptr; - const uint64_t ne2 = use_src2 ? src2->ne[0] * src2->ne[1] : 0; - - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op); + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, dst, op); ggml_vk_func_t op_func; if (pipeline == nullptr) { @@ -3955,18 +3949,15 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; - ggml_tensor_extra_gpu * extra_src2 = use_src2 ? (ggml_tensor_extra_gpu *) src2->extra : nullptr; vk_buffer d_X = nullptr; size_t x_buf_offset = 0; vk_buffer d_Y = nullptr; size_t y_buf_offset = 0; vk_buffer d_Z = nullptr; - size_t z_buf_offset = 0; bool src0_uma = false; bool src1_uma = false; - bool src2_uma = false; if (ctx->device->uma) { ggml_vk_host_get(ctx, src0->data, d_X, x_buf_offset); @@ -3975,15 +3966,10 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_host_get(ctx, src1->data, d_Y, y_buf_offset); src1_uma = d_Y != nullptr; } - if (use_src2) { - ggml_vk_host_get(ctx, src1->data, d_Z, z_buf_offset); - src2_uma = d_Z != nullptr; - } } uint64_t x_sz = ggml_vk_align_size(ggml_type_size(src0->type)/ggml_blck_size(src0->type) * ne0, ctx->device->properties.limits.minStorageBufferOffsetAlignment); uint64_t y_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * ne1, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : 0; - uint64_t z_sz = use_src2 ? ggml_vk_align_size(ggml_type_size(src2->type) * ne2, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : 0; uint64_t d_sz = ggml_type_size(dst->type) * ne0; vk_buffer d_D = extra->buffer_gpu.lock(); @@ -4007,12 +3993,6 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c GGML_ASSERT(d_Y != nullptr); } - if (use_src2 && !src2_uma) { - d_Z = extra_src2->buffer_gpu.lock(); - z_buf_offset = extra_src2->offset; - GGML_ASSERT(d_Z != nullptr); - } - if (op_supports_incontiguous) { x_sz = ggml_nbytes(src0); y_sz = use_src1 ? ggml_nbytes(src1) : 0; @@ -4046,7 +4026,10 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c elements = { (uint32_t)ggml_nrows(src0), (uint32_t)ne00, 1 }; break; case GGML_OP_GET_ROWS: - elements = { (uint32_t)ne00, (uint32_t)ne10, (uint32_t)(ne11 * ne12) }; + elements = { (uint32_t)ne00, (uint32_t)ne10, (uint32_t)(ne11 * ne12) }; + break; + case GGML_OP_ARGSORT: + elements = { (uint32_t)ne00, (uint32_t)ggml_nrows(src0), 1 }; break; default: elements = { (uint32_t)ggml_nelements(src0), 1, 1 }; @@ -4066,7 +4049,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } if (op == GGML_OP_SOFT_MAX) { - // Empty src1 and src2 are possible on soft_max, but the shader needs buffers + // Empty src1 is possible on soft_max, but the shader needs a buffer vk_subbuffer subbuf_y; if (use_src1) { subbuf_y = { d_Y, y_buf_offset, y_sz }; @@ -4074,15 +4057,8 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c subbuf_y = { d_X, 0, d_X->size }; } - vk_subbuffer subbuf_z; - if (use_src2) { - subbuf_z = { d_Z, z_buf_offset, z_sz }; - } else { - subbuf_z = { d_X, 0, d_X->size }; - } - ggml_vk_sync_buffers(subctx); - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, subbuf_y, subbuf_z, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, subbuf_y, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } else if (use_src1) { ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); @@ -4099,13 +4075,13 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } } else { GGML_ASSERT(op != GGML_OP_SOFT_MAX); + GGML_ASSERT(op != GGML_OP_ARGSORT); ggml_pipeline_allocate_descriptor_sets(ctx, pipeline, ne02 * ne03); switch (dst->op) { case GGML_OP_NORM: case GGML_OP_RMS_NORM: - case GGML_OP_SOFT_MAX: elements = { (uint32_t)ne01, 1, 1 }; break; case GGML_OP_DIAG_MASK_INF: @@ -4145,7 +4121,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } static void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -4153,7 +4129,7 @@ static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context * subctx, const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_GET_ROWS, { + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_GET_ROWS, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, @@ -4168,7 +4144,7 @@ static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ADD, { + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ADD, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, @@ -4183,7 +4159,7 @@ static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_MUL, { + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_MUL, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, @@ -4198,7 +4174,7 @@ static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context * subctx, co const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_SCALE, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SCALE, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -4211,7 +4187,7 @@ static void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_SQR, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SQR, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -4225,7 +4201,7 @@ static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context * subctx, co const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_CLAMP, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CLAMP, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -4240,7 +4216,7 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t dst_type_size = ggml_type_size(dst->type); const uint32_t d_offset = (extra->offset % ctx->device->properties.limits.minStorageBufferOffsetAlignment) / dst_type_size; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_CPY, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CPY, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -4252,24 +4228,24 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context * subctx, cons static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); } static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); } static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); } static void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); } -static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { +static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; float scale = op_params[0]; @@ -4285,13 +4261,9 @@ static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); -#pragma message("TODO: src2 is no longer used in soft_max - should be removed and ALiBi calculation should be updated") -#pragma message("ref: https://github.com/ggerganov/llama.cpp/pull/7192") - - ggml_vk_op_f32(ctx, subctx, src0, src1, src2, dst, GGML_OP_SOFT_MAX, { + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_SOFT_MAX, { ncols, src1 != nullptr ? nrows_y : (uint32_t)0, - src2 != nullptr ? (uint32_t)1 : (uint32_t)0, scale, max_bias, m0, m1, n_head_log2, @@ -4321,15 +4293,39 @@ static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, con if (is_neox) { const float theta_scale = powf(freq_base, -2.0f/n_dims); const float inv_ndims = -1.0f / n_dims; - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1], 0.0f, 0.0f}, theta_scale, inv_ndims }); + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { + (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], + freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1], 0.0f, 0.0f}, theta_scale, inv_ndims + }); } else { - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1], 0.0f, 0.0f} }); + ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { + (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], + freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1], 0.0f, 0.0f} + }); } } static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_ARGSORT, { (uint32_t)src0->ne[0], ((ggml_sort_order) op_params[0]) == GGML_SORT_ORDER_ASC }); + + uint32_t ncols = src0->ne[0]; + + uint32_t ncols_pad = 1; + while (ncols_pad < ncols) { + ncols_pad *= 2; + } + + GGML_ASSERT(ncols_pad <= 1024); + + std::cerr << "ncols=" << ncols << " ncols_pad=" << ncols_pad << " ascending=" << op_params[0] << std::endl; + + std::cerr << ((ggml_sort_order) op_params[0]) << " " << GGML_SORT_ORDER_ASC << std::endl; + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_ARGSORT, { + ncols, + ncols_pad, + op_params[0], + }); } #ifdef GGML_VULKAN_RUN_TESTS @@ -5432,7 +5428,6 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod const ggml_tensor * src0 = node->src[0]; const ggml_tensor * src1 = node->src[1]; - const ggml_tensor * src2 = node->src[2]; ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) node->extra; @@ -5547,7 +5542,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod break; case GGML_OP_SOFT_MAX: - ggml_vk_soft_max(ctx, ctx->compute_ctx, src0, src1, src2, node); + ggml_vk_soft_max(ctx, ctx->compute_ctx, src0, src1, node); break; case GGML_OP_ROPE: @@ -6548,7 +6543,7 @@ static void ggml_vk_print_graph_origin(const ggml_tensor * tensor, std::vectortype != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) { + if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16 && tensor->type != GGML_TYPE_I32) { return; } i0 = std::max(i0, 5); @@ -6569,6 +6564,8 @@ static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * d val = *(const float *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]); } else if (tensor->type == GGML_TYPE_F16) { val = ggml_fp16_to_fp32(*(const ggml_fp16_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0])); + } else if (tensor->type == GGML_TYPE_I32) { + val = *(const int32_t *) ((const char *) data + i3*tensor->nb[3] + i2*tensor->nb[2] + idx1*tensor->nb[1] + idx0*tensor->nb[0]); } else { GGML_ASSERT(false); } @@ -6671,7 +6668,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; - ggml_tensor * src2 = tensor->src[2]; struct ggml_init_params iparams = { /*.mem_size =*/ 1024*1024*1024, @@ -6798,66 +6794,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src1", src1_clone); } - if (src2 != nullptr) { - src2_clone = ggml_dup_tensor(ggml_ctx, src2); - - src2_size = ggml_nbytes(src2); - - src2_buffer = malloc(src2_size); - src2_clone->data = src2_buffer; - if (src2->backend == GGML_BACKEND_TYPE_CPU) { - memcpy(src2_clone->data, src2->data, src2_size); - memcpy(src2_clone->nb, src2->nb, sizeof(size_t) * GGML_MAX_DIMS); - } else if (src2->backend == GGML_BACKEND_TYPE_GPU) { - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src2->extra; - vk_buffer buf = extra->buffer_gpu.lock(); - uint64_t offset = extra->offset; - if (!ggml_is_contiguous(src2) && ggml_vk_dim01_contiguous(src2)) { - for (int i3 = 0; i3 < src2->ne[3]; i3++) { - for (int i2 = 0; i2 < src2->ne[2]; i2++) { - const int idx = i3*src2->ne[2] + i2; - ggml_vk_buffer_read(ctx, buf, offset + idx * src2->nb[2], ((char *)src2_clone->data + idx * src2_clone->nb[2]), src2->ne[1] * src2->nb[1]); - } - } - - src2_clone->nb[0] = src2->nb[0]; - src2_clone->nb[1] = src2->nb[1]; - for (int i = 2; i < GGML_MAX_DIMS; i++) { - src2_clone->nb[i] = src2_clone->nb[i - 1]*src2_clone->ne[i - 1]; - } - } else { - if (offset + src2_size >= buf->size) { - src2_size = buf->size - offset; - } - ggml_vk_buffer_read(ctx, buf, offset, src2_clone->data, src2_size); - memcpy(src2_clone->nb, src2->nb, sizeof(size_t) * GGML_MAX_DIMS); - } - } else { - GGML_ASSERT(false); - } - - if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { - ggml_vk_print_tensor(ctx, src2, "src2"); - std::cerr << "TENSOR CHECK: " << ggml_op_name(src2_clone->op) << " (check " << check_counter << ")" << std::endl; - std::cerr << "src2_clone=" << tensor << " src2_clone->backend: " << src2_clone->backend << " src2_clone->type: " << ggml_type_name(src2_clone->type) << " ne0=" << src2_clone->ne[0] << " nb0=" << src2_clone->nb[0] << " ne1=" << src2_clone->ne[1] << " nb1=" << src2_clone->nb[1] << " ne2=" << src2_clone->ne[2] << " nb2=" << src2_clone->nb[2] << " ne3=" << src2_clone->ne[3] << " nb3=" << src2_clone->nb[3] << std::endl; - if (src2->src[0] != nullptr) { - std::cerr << "src2->src[0]=" << src2->src[0] << " op=" << ggml_op_name(src2->src[0]->op) << " type=" << ggml_type_name(src2->src[0]->type) << " backend=" << src2->src[0]->backend << " ne0=" << src2->src[0]->ne[0] << " nb0=" << src2->src[0]->nb[0] << " ne1=" << src2->src[0]->ne[1] << " nb1=" << src2->src[0]->nb[1] << " ne2=" << src2->src[0]->ne[2] << " nb2=" << src2->src[0]->nb[2] << " ne3=" << src2->src[0]->ne[3] << " nb3=" << src2->src[0]->nb[3] << std::endl; - } - if (src2->src[1] != nullptr) { - std::cerr << "src2->src[1]=" << src2->src[1] << " op=" << ggml_op_name(src2->src[1]->op) << " type=" << ggml_type_name(src2->src[1]->type) << " backend=" << src2->src[1]->backend << " ne0=" << src2->src[1]->ne[0] << " nb0=" << src2->src[1]->nb[0] << " ne1=" << src2->src[1]->ne[1] << " nb1=" << src2->src[1]->nb[1] << " ne2=" << src2->src[1]->ne[2] << " nb2=" << src2->src[1]->nb[2] << " ne3=" << src2->src[1]->ne[3] << " nb3=" << src2->src[1]->nb[3] << std::endl; - } - std::cerr << std::endl << "Result:" << std::endl; - ggml_vk_print_tensor_area(src2_clone, src2_clone->data, 5, 5, 0, 0); - std::cerr << std::endl; - std::cerr << std::endl << "Result:" << std::endl; - ggml_vk_print_tensor_area(src2_clone, src2_clone->data, 5, 5, 1, 0); - std::cerr << std::endl; - std::vector done; - ggml_vk_print_graph_origin(src2_clone, done); - } - - ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src2", src2_clone); - } if (tensor->op == GGML_OP_MUL_MAT) { tensor_clone = ggml_mul_mat(ggml_ctx, src0_clone, src1_clone); @@ -6877,7 +6813,7 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ tensor_clone = ggml_rms_norm(ggml_ctx, src0_clone, *(float *)tensor->op_params); } else if (tensor->op == GGML_OP_SOFT_MAX) { if (src1 != nullptr) { - tensor_clone = ggml_soft_max_ext(ggml_ctx, src0_clone, src1_clone, src2_clone, ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); + tensor_clone = ggml_soft_max_ext(ggml_ctx, src0_clone, src1_clone, ((float *)tensor->op_params)[0], ((float *)tensor->op_params)[1]); } else { tensor_clone = ggml_soft_max(ggml_ctx, src0_clone); } @@ -6964,9 +6900,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ if (src1 != nullptr) { free(src1_buffer); } - if (src2 != nullptr) { - free(src2_buffer); - } ggml_free(ggml_ctx); } @@ -7026,8 +6959,11 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ } else if (tensor->type == GGML_TYPE_F16) { correct = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0])); result = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0])); + } else if (tensor->type == GGML_TYPE_I32) { + correct = *(int32_t *) ((char *) comp_result + i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]); + result = *(int32_t *) ((char *) tensor_data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]); } else { - std::cerr << "comp_size=" << comp_size << " but required is " << (i3*comp_nb[3] + i2*comp_nb[2] + i1*comp_nb[1] + i0*comp_nb[0]) << std::endl; + std::cerr << "Results check not implemented for type " << ggml_type_name(tensor->type) << std::endl; } } else { std::cerr << "Missing debug code for type " << ggml_type_name(tensor->type) << std::endl; diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index 162cf5c6e..3c22a986c 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -2432,7 +2432,6 @@ layout (push_constant) uniform parameter { uint KX; uint KY; - uint KZ; float scale; float max_bias; float m0; @@ -2449,8 +2448,7 @@ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; layout (binding = 1) readonly buffer Y {B_TYPE data_b[];}; -layout (binding = 2) readonly buffer Z {C_TYPE data_c[];}; -layout (binding = 3) buffer D {D_TYPE data_d[];}; +layout (binding = 2) buffer D {D_TYPE data_d[];}; shared FLOAT_TYPE vals[BLOCK_SIZE]; @@ -2459,7 +2457,7 @@ void main() { const uint rowx = gl_WorkGroupID.x; const uint rowy = rowx % p.KY; - float slope = 0.0f; + float slope = 1.0f; // ALiBi if (p.max_bias > 0.0f) { @@ -2472,11 +2470,18 @@ void main() { } // Find max - vals[tid] = uintBitsToFloat(0xFF800000); + FLOAT_TYPE max_val = uintBitsToFloat(0xFF800000); - [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { - vals[tid] = max(vals[tid], FLOAT_TYPE(data_a[rowx * p.KX + col]) * p.scale + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) + (p.KZ > 0 ? slope * FLOAT_TYPE(data_c[col]) : 0.0f)); + [[unroll]] for (uint col0 = 0; col0 < p.KX; col0 += BLOCK_SIZE) { + const uint col = col0 + tid; + + if (col >= p.KX) { + break; + } + + max_val = max(max_val, FLOAT_TYPE(data_a[rowx * p.KX + col]) * p.scale + (p.KY > 0 ? slope * FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f))); } + vals[tid] = max_val; barrier(); [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { @@ -2486,15 +2491,21 @@ void main() { barrier(); } - const FLOAT_TYPE max_val = vals[0]; + max_val = vals[0]; barrier(); // Sum up values vals[tid] = FLOAT_TYPE(0.0f); - [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + [[unroll]] for (uint col0 = 0; col0 < p.KX; col0 += BLOCK_SIZE) { + const uint col = col0 + tid; + + if (col >= p.KX) { + break; + } + const uint i = rowx * p.KX + col; - const FLOAT_TYPE val = exp(FLOAT_TYPE(data_a[i]) * p.scale + (p.KY > 0 ? FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) - max_val); + const FLOAT_TYPE val = exp(FLOAT_TYPE(data_a[i]) * p.scale + (p.KY > 0 ? slope * FLOAT_TYPE(data_b[rowy * p.KX + col]) : FLOAT_TYPE(0.0f)) - max_val); vals[tid] += val; data_d[i] = D_TYPE(val); } @@ -2509,7 +2520,13 @@ void main() { const D_TYPE divisor = D_TYPE(vals[0]); - [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { + [[unroll]] for (uint col0 = 0; col0 < p.KX; col0 += BLOCK_SIZE) { + const uint col = col0 + tid; + + if (col >= p.KX) { + break; + } + data_d[rowx*p.KX + col] /= divisor; } } @@ -2672,20 +2689,26 @@ argsort_src = """ #extension GL_EXT_shader_16bit_storage : require -layout(local_size_x = 1024, local_size_y = 1, local_size_z = 1) in; +#define BLOCK_SIZE 1024 +#define ASC 0 + +layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; layout (binding = 1) buffer D {int data_d[];}; layout (push_constant) uniform parameter { uint ncols; - bool ascending; + uint ncols_pad; + uint order; } p; +shared int dst_row[BLOCK_SIZE]; + void swap(uint idx0, uint idx1) { - int tmp = data_d[idx0]; - data_d[idx0] = data_d[idx1]; - data_d[idx1] = tmp; + int tmp = dst_row[idx0]; + dst_row[idx0] = dst_row[idx1]; + dst_row[idx1] = tmp; } void main() { @@ -2693,36 +2716,45 @@ void main() { const int col = int(gl_LocalInvocationID.x); const uint row = gl_WorkGroupID.y; - if (col >= p.ncols) { + if (col >= p.ncols_pad) { return; } - const uint a_idx = row * p.ncols; - const uint d_idx = row * p.ncols; + const uint row_offset = row * p.ncols; // initialize indices - if (col < p.ncols) { - data_d[col] = col; - } + dst_row[col] = col; barrier(); - for (uint k = 2; k <= p.ncols; k *= 2) { + for (uint k = 2; k <= p.ncols_pad; k *= 2) { for (uint j = k / 2; j > 0; j /= 2) { const uint ixj = col ^ j; if (ixj > col) { if ((col & k) == 0) { - if (p.ascending ? data_a[a_idx + data_d[d_idx + col]] > data_a[a_idx + data_d[d_idx + ixj]] : data_a[a_idx + data_d[d_idx + col]] < data_a[a_idx + data_d[d_idx + ixj]]) { - swap(d_idx + col, d_idx + ixj); + if (dst_row[col] >= p.ncols || + (dst_row[ixj] < p.ncols && (p.order == ASC ? + data_a[row_offset + dst_row[col]] > data_a[row_offset + dst_row[ixj]] : + data_a[row_offset + dst_row[col]] < data_a[row_offset + dst_row[ixj]])) + ) { + swap(col, ixj); } } else { - if (p.ascending ? data_a[a_idx + data_d[d_idx + col]] < data_a[a_idx + data_d[d_idx + ixj]] : data_a[a_idx + data_d[d_idx + col]] > data_a[a_idx + data_d[d_idx + ixj]]) { - swap(d_idx + col, d_idx + ixj); + if (dst_row[ixj] >= p.ncols || + (dst_row[col] < p.ncols && (p.order == ASC ? + data_a[row_offset + dst_row[col]] < data_a[row_offset + dst_row[ixj]] : + data_a[row_offset + dst_row[col]] > data_a[row_offset + dst_row[ixj]])) + ) { + swap(col, ixj); } } } barrier(); } } + + if (col < p.ncols) { + data_d[row_offset + col] = dst_row[col]; + } } """ From ca57e0f35e33f714b9a6c2c4482b87bfe059c819 Mon Sep 17 00:00:00 2001 From: strawberrymelonpanda <152940198+strawberrymelonpanda@users.noreply.github.com> Date: Sat, 18 May 2024 00:57:08 -0700 Subject: [PATCH 061/137] perplexity : ndot progress and show stats with < 100 tasks (#7348) Fix floating point error with ndot printing, allow end stats on lower task numbers if multiple-choice tasks. --- examples/perplexity/perplexity.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index db6e0949d..bae014e6f 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -1425,7 +1425,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params // Use all tasks tasks.resize(n_task); printf("%s: reading tasks", __func__); - int n_dot = n_task/100; + int n_dot = std::max((int) n_task/100, 1); int i = 0; for (auto& task : tasks) { ++i; @@ -1675,7 +1675,7 @@ static void multiple_choice_score(llama_context * ctx, const gpt_params & params llama_batch_free(batch); - if (n_done < 100) return; + if (n_done < 100 && (params.multiple_choice_tasks != 0 && params.multiple_choice_tasks < (size_t)n_task)) return; float p = 1.f*n_correct/n_done; float sigma = sqrt(p*(1-p)/(n_done-1)); From 0f98acfac6cc561dc57586bfff778405e42b576b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20R=C3=B6cker?= Date: Sat, 18 May 2024 10:04:55 +0200 Subject: [PATCH 062/137] llama : add support for larger Granite Code Models (20B, 34B) (#7324) Tie the weights for ARCH_STARCODER to support the larger Granite code models. Partially addresses ggerganov/issues/7116 There still remains to be a few things to fix. Currently requires `--override-kv tokenizer.ggml.add_bos_token=bool:false` --- llama.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index b752ddc6b..2b91eec86 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5188,7 +5188,14 @@ static bool llm_load_tensors( { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + if (!model.output) { + // needs to be on GPU + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + ml.n_created--; // artificial tensor + ml.size_data += ggml_nbytes(model.output); + } + } for (int i = 0; i < n_layer; ++i) { From d233b507cd19fcc2d8d8963ecc6a3eb7a33f2ecc Mon Sep 17 00:00:00 2001 From: Engininja2 <139037756+Engininja2@users.noreply.github.com> Date: Sat, 18 May 2024 02:05:17 -0600 Subject: [PATCH 063/137] cuda : add half2 __shfl_xor() for ROCm 5.5 (#7263) --- ggml-cuda/common.cuh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ggml-cuda/common.cuh b/ggml-cuda/common.cuh index b6f0bc36a..784792ba0 100644 --- a/ggml-cuda/common.cuh +++ b/ggml-cuda/common.cuh @@ -315,6 +315,20 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) { #endif return c; } + +#if defined(__HIP_PLATFORM_AMD__) && HIP_VERSION < 50600000 +// __shfl_xor() for half2 was added in ROCm 5.6 +static __device__ __forceinline__ half2 __shfl_xor(half2 var, int laneMask, int width) { + typedef union half2_b32 { + half2 val; + int b32; + } half2_b32_t; + half2_b32_t tmp; + tmp.val = var; + tmp.b32 = __shfl_xor(tmp.b32, laneMask, width); + return tmp.val; +} +#endif // defined(__HIP_PLATFORM_AMD__) && HIP_VERSION < 50600000 #endif // defined(GGML_USE_HIPBLAS) #define FP16_AVAILABLE (defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) || __CUDA_ARCH__ >= CC_PASCAL From cb42c294279bc4a0a4e926a7b5a5568049f12fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 18 May 2024 11:10:47 +0200 Subject: [PATCH 064/137] server: correct --threads documentation [no ci] (#7362) --- examples/server/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 13cbdcd2c..4f3262cdd 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -18,8 +18,8 @@ The project is under active development, and we are [looking for feedback and co **Command line options:** - `-v`, `--verbose`: Enable verbose server output. When using the `/completion` endpoint, this includes the tokenized prompt, the full request and the full response. -- `-t N`, `--threads N`: Set the number of threads to use during generation. Not used if model layers are offloaded to GPU. The server is using batching. This parameter is used only if one token is to be processed on CPU backend. -- `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. If not specified, the number of threads will be set to the number of threads used for generation. Not used if model layers are offloaded to GPU. +- `-t N`, `--threads N`: Set the number of threads to use by CPU layers during generation. Not used by model layers that are offloaded to GPU. This option has no effect when using the maximum number of GPU layers. Default: `std::thread::hardware_concurrency()` (number of CPU cores). +- `-tb N, --threads-batch N`: Set the number of threads to use by CPU layers during batch and prompt processing (>= 32 tokens). This option has no effect if a GPU is available. Default: `--threads`. - `--threads-http N`: Number of threads in the http server pool to process requests. Default: `max(std::thread::hardware_concurrency() - 1, --parallel N + 2)` - `-m FNAME`, `--model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.gguf`). - `-mu MODEL_URL --model-url MODEL_URL`: Specify a remote http url to download the file. Default: unused From 133d99c59980139f5bb75922c8b5fca67d7ba9b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sat, 18 May 2024 12:36:25 +0200 Subject: [PATCH 065/137] CUDA: deduplicate FlashAttention code (#7352) --- ggml-cuda/common.cuh | 11 ++ ggml-cuda/fattn-common.cuh | 115 +++++++++++++++++++ ggml-cuda/fattn-tile-f16.cu | 135 +++++----------------- ggml-cuda/fattn-tile-f32.cu | 134 +++++----------------- ggml-cuda/fattn-vec-f16.cu | 215 ++++++++++-------------------------- ggml-cuda/fattn-vec-f32.cu | 170 ++++++---------------------- ggml-cuda/fattn.cu | 177 ++++++++--------------------- ggml-cuda/softmax.cu | 13 +-- 8 files changed, 316 insertions(+), 654 deletions(-) diff --git a/ggml-cuda/common.cuh b/ggml-cuda/common.cuh index 784792ba0..8f6fd71cf 100644 --- a/ggml-cuda/common.cuh +++ b/ggml-cuda/common.cuh @@ -477,6 +477,17 @@ static const __device__ int8_t kvalues_iq4nl[16] = {-127, -104, -83, -65, -49, - typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v); +static __device__ __forceinline__ float get_alibi_slope( + const float max_bias, const uint32_t h, const uint32_t n_head_log2, const float m0, const float m1 +) { + if (max_bias <= 0.0f) { + return 1.0f; + } + const float base = h < n_head_log2 ? m0 : m1; + const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; + + return powf(base, exph); +} ////////////////////// diff --git a/ggml-cuda/fattn-common.cuh b/ggml-cuda/fattn-common.cuh index 33f640691..1dd519bde 100644 --- a/ggml-cuda/fattn-common.cuh +++ b/ggml-cuda/fattn-common.cuh @@ -1,7 +1,44 @@ +#include "common.cuh" + +#include + #define FATTN_KQ_STRIDE 256 #define HALF_MAX_HALF __float2half(65504.0f/2) // Use neg. of this instead of -INFINITY to initialize KQ max vals to avoid NaN upon subtraction. #define SOFTMAX_FTZ_THRESHOLD -20.0f // Softmax exp. of values smaller than this are flushed to zero to avoid NaNs. +typedef void (* fattn_kernel_t)( + const char * __restrict__ Q, + const char * __restrict__ K, + const char * __restrict__ V, + const char * __restrict__ mask, + float * __restrict__ dst, + float2 * __restrict__ dst_meta, + const float scale, + const float max_bias, + const float m0, + const float m1, + const uint32_t n_head_log2, + const int ne00, + const int ne01, + const int ne02, + const int ne03, + const int ne10, + const int ne11, + const int ne12, + const int ne13, + const int ne31, + const int nb31, + const int nb01, + const int nb02, + const int nb03, + const int nb11, + const int nb12, + const int nb13, + const int ne0, + const int ne1, + const int ne2, + const int ne3); + template // D == head size #if !(defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)) __launch_bounds__(D, 1) @@ -45,3 +82,81 @@ static __global__ void flash_attn_combine_results( dst[blockIdx.y*D + tid] = VKQ_numerator / VKQ_denominator; } + +template +void launch_fattn(ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, int nwarps, int cols_per_block) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + + const ggml_tensor * mask = dst->src[3]; + + ggml_tensor * KQV = dst; + + GGML_ASSERT(Q->type == GGML_TYPE_F32); + GGML_ASSERT(K->type == GGML_TYPE_F16); + GGML_ASSERT(V->type == GGML_TYPE_F16); + GGML_ASSERT(KQV->type == GGML_TYPE_F32); + + GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16); + GGML_ASSERT(!mask || mask->ne[1] >= GGML_PAD(Q->ne[1], 16) && + "the Flash-Attention CUDA kernel requires the mask to be padded to 16 and at least n_queries big"); + + GGML_ASSERT(K->ne[1] % FATTN_KQ_STRIDE == 0 && "Incorrect KV cache padding."); + + ggml_cuda_pool & pool = ctx.pool(); + cudaStream_t main_stream = ctx.stream(); + + ggml_cuda_pool_alloc dst_tmp(pool); + ggml_cuda_pool_alloc dst_tmp_meta(pool); + + if (parallel_blocks > 1) { + dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); + dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); + } + + const dim3 block_dim(WARP_SIZE, nwarps, 1); + const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); + const int shmem = 0; + + float scale = 1.0f; + float max_bias = 0.0f; + + memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); + memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); + + const uint32_t n_head = Q->ne[2]; + const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); + + const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); + const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + + fattn_kernel<<>>( + (const char *) Q->data, + (const char *) K->data, + (const char *) V->data, + mask ? ((const char *) mask->data) : nullptr, + (parallel_blocks) == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, + scale, max_bias, m0, m1, n_head_log2, + Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], + K->ne[0], K->ne[1], K->ne[2], K->ne[3], + mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, + Q->nb[1], Q->nb[2], Q->nb[3], + K->nb[1], K->nb[2], K->nb[3], + KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] + ); + CUDA_CHECK(cudaGetLastError()); + + if ((parallel_blocks) == 1) { + return; + } + + const dim3 block_dim_combine(D, 1, 1); + const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); + const int shmem_combine = 0; + + flash_attn_combine_results + <<>> + (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); + CUDA_CHECK(cudaGetLastError()); +} diff --git a/ggml-cuda/fattn-tile-f16.cu b/ggml-cuda/fattn-tile-f16.cu index d2a1077ed..4a07ac6ad 100644 --- a/ggml-cuda/fattn-tile-f16.cu +++ b/ggml-cuda/fattn-tile-f16.cu @@ -54,17 +54,8 @@ static __global__ void flash_attn_tile_ext_f16( const int stride_KV2 = nb11 / sizeof(half2); - half slopeh = __float2half(1.0f); - - // ALiBi - if (max_bias > 0.0f) { - const uint32_t h = blockIdx.y; - - const float base = h < n_head_log2 ? m0 : m1; - const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; - - slopeh = __float2half(powf(base, exph)); - } + const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const half slopeh = __float2half(slopef); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); @@ -272,124 +263,50 @@ static __global__ void flash_attn_tile_ext_f16( #endif // FP16_AVAILABLE } -template void launch_fattn_tile_f16( - const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, - ggml_cuda_pool & pool, cudaStream_t main_stream -) { - ggml_cuda_pool_alloc dst_tmp(pool); - ggml_cuda_pool_alloc dst_tmp_meta(pool); - - if (parallel_blocks > 1) { - dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); - dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); +template +void launch_fattn_tile_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + switch (Q->ne[0]) { + case 64: { + constexpr int D = 64; + constexpr int nwarps = 8; + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + case 128: { + constexpr int D = 128; + constexpr int nwarps = 8; + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + default: { + GGML_ASSERT(false && "FlashAttention without tensor cores only supports head sizes 64 and 128."); + } break; } - - constexpr int nwarps = 8; - const dim3 block_dim(WARP_SIZE, nwarps, 1); - const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); - const int shmem = 0; - - float scale = 1.0f; - float max_bias = 0.0f; - - memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); - - const uint32_t n_head = Q->ne[2]; - const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); - - const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - - flash_attn_tile_ext_f16 - <<>> ( - (const char *) Q->data, - (const char *) K->data, - (const char *) V->data, - mask ? ((const char *) mask->data) : nullptr, - parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, - scale, max_bias, m0, m1, n_head_log2, - Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], - K->ne[0], K->ne[1], K->ne[2], K->ne[3], - mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, - Q->nb[1], Q->nb[2], Q->nb[3], - K->nb[1], K->nb[2], K->nb[3], - KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] - ); - CUDA_CHECK(cudaGetLastError()); - - if (parallel_blocks == 1) { - return; - } - - const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); - const int shmem_combine = 0; - - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); - CUDA_CHECK(cudaGetLastError()); } void ggml_cuda_flash_attn_ext_tile_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * Q = dst->src[0]; - const ggml_tensor * K = dst->src[1]; - const ggml_tensor * V = dst->src[2]; - - const ggml_tensor * mask = dst->src[3]; - - ggml_tensor * KQV = dst; + const ggml_tensor * KQV = dst; + const ggml_tensor * Q = dst->src[0]; const int32_t precision = KQV->op_params[2]; GGML_ASSERT(precision == GGML_PREC_DEFAULT); - GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); if (Q->ne[1] <= 16) { constexpr int cols_per_block = 16; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_tile_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_tile_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_tile_f16_64_128(ctx, dst); return; } if (Q->ne[1] <= 32) { constexpr int cols_per_block = 32; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_tile_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_tile_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_tile_f16_64_128(ctx, dst); return; } constexpr int cols_per_block = 32; constexpr int parallel_blocks = 1; - switch (Q->ne[0]) { - case 64: - launch_fattn_tile_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_tile_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_tile_f16_64_128(ctx, dst); } diff --git a/ggml-cuda/fattn-tile-f32.cu b/ggml-cuda/fattn-tile-f32.cu index 176895edd..130e7cbdb 100644 --- a/ggml-cuda/fattn-tile-f32.cu +++ b/ggml-cuda/fattn-tile-f32.cu @@ -53,17 +53,7 @@ static __global__ void flash_attn_tile_ext_f32( const int stride_KV2 = nb11 / sizeof(half2); - float slope = 1.0f; - - // ALiBi - if (max_bias > 0.0f) { - const uint32_t h = blockIdx.y; - - const float base = h < n_head_log2 ? m0 : m1; - const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; - - slope = powf(base, exph); - } + const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); @@ -270,124 +260,50 @@ static __global__ void flash_attn_tile_ext_f32( } } -template void launch_fattn_tile_f32( - const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, - ggml_cuda_pool & pool, cudaStream_t main_stream -) { - ggml_cuda_pool_alloc dst_tmp(pool); - ggml_cuda_pool_alloc dst_tmp_meta(pool); - - if (parallel_blocks > 1) { - dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); - dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); +template +void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + switch (Q->ne[0]) { + case 64: { + constexpr int D = 64; + constexpr int nwarps = 8; + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + case 128: { + constexpr int D = 128; + constexpr int nwarps = 8; + fattn_kernel_t fattn_kernel = flash_attn_tile_ext_f32; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + default: { + GGML_ASSERT(false && "FlashAttention without tensor cores only supports head sizes 64 and 128."); + } break; } - - constexpr int nwarps = 8; - const dim3 block_dim(WARP_SIZE, nwarps, 1); - const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); - const int shmem = 0; - - float scale = 1.0f; - float max_bias = 0.0f; - - memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); - - const uint32_t n_head = Q->ne[2]; - const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); - - const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - - flash_attn_tile_ext_f32 - <<>> ( - (const char *) Q->data, - (const char *) K->data, - (const char *) V->data, - mask ? ((const char *) mask->data) : nullptr, - parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, - scale, max_bias, m0, m1, n_head_log2, - Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], - K->ne[0], K->ne[1], K->ne[2], K->ne[3], - mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, - Q->nb[1], Q->nb[2], Q->nb[3], - K->nb[1], K->nb[2], K->nb[3], - KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] - ); - CUDA_CHECK(cudaGetLastError()); - - if (parallel_blocks == 1) { - return; - } - - const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); - const int shmem_combine = 0; - - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); - CUDA_CHECK(cudaGetLastError()); } void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * Q = dst->src[0]; - const ggml_tensor * K = dst->src[1]; - const ggml_tensor * V = dst->src[2]; - - const ggml_tensor * mask = dst->src[3]; - - ggml_tensor * KQV = dst; + const ggml_tensor * KQV = dst; + const ggml_tensor * Q = dst->src[0]; const int32_t precision = KQV->op_params[2]; GGML_ASSERT(precision == GGML_PREC_DEFAULT); - GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); if (Q->ne[1] <= 16) { constexpr int cols_per_block = 16; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_tile_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_tile_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_tile_f32_64_128(ctx, dst); return; } if (Q->ne[1] <= 32) { constexpr int cols_per_block = 32; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_tile_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_tile_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_tile_f32_64_128(ctx, dst); return; } constexpr int cols_per_block = 32; constexpr int parallel_blocks = 1; - switch (Q->ne[0]) { - case 64: - launch_fattn_tile_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_tile_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_tile_f32_64_128(ctx, dst); } diff --git a/ggml-cuda/fattn-vec-f16.cu b/ggml-cuda/fattn-vec-f16.cu index a18be5ddc..54e1ac5d1 100644 --- a/ggml-cuda/fattn-vec-f16.cu +++ b/ggml-cuda/fattn-vec-f16.cu @@ -53,17 +53,8 @@ static __global__ void flash_attn_vec_ext_f16( const int stride_KV = nb11 / sizeof(half); const int stride_KV2 = nb11 / sizeof(half2); - half slopeh = __float2half(1.0f); - - // ALiBi - if (max_bias > 0.0f) { - const uint32_t h = blockIdx.y; - - const float base = h < n_head_log2 ? m0 : m1; - const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; - - slopeh = __float2half(powf(base, exph)); - } + const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const half slopeh = __float2half(slopef); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); constexpr int nwarps = D / WARP_SIZE; @@ -232,196 +223,104 @@ static __global__ void flash_attn_vec_ext_f16( dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; } - if (parallel_blocks != 1 && threadIdx.x < ncols) { - dst_meta[(ic0 + threadIdx.x)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[threadIdx.x], kqsum[threadIdx.x]); + if (parallel_blocks != 1 && tid < ncols) { + dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); } #else NO_DEVICE_CODE; #endif // FP16_AVAILABLE } -template void launch_fattn_vec_f16( - const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, - ggml_cuda_pool & pool, cudaStream_t main_stream -) { - ggml_cuda_pool_alloc dst_tmp(pool); - ggml_cuda_pool_alloc dst_tmp_meta(pool); - - if (parallel_blocks > 1) { - dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); - dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); - } - - constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE; - const dim3 block_dim(WARP_SIZE, nwarps, 1); - const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); - const int shmem = 0; - - float scale = 1.0f; - float max_bias = 0.0f; - - memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); - - const uint32_t n_head = Q->ne[2]; - const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); - - const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - - flash_attn_vec_ext_f16 - <<>> ( - (const char *) Q->data, - (const char *) K->data, - (const char *) V->data, - mask ? ((const char *) mask->data) : nullptr, - parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, - scale, max_bias, m0, m1, n_head_log2, - Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], - K->ne[0], K->ne[1], K->ne[2], K->ne[3], - mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, - Q->nb[1], Q->nb[2], Q->nb[3], - K->nb[1], K->nb[2], K->nb[3], - KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] - ); - CUDA_CHECK(cudaGetLastError()); - - if (parallel_blocks == 1) { - return; - } - - const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); - const int shmem_combine = 0; - - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); - CUDA_CHECK(cudaGetLastError()); -} - void ggml_cuda_flash_attn_ext_vec_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * Q = dst->src[0]; - const ggml_tensor * K = dst->src[1]; - const ggml_tensor * V = dst->src[2]; - - const ggml_tensor * mask = dst->src[3]; - ggml_tensor * KQV = dst; + ggml_tensor * Q = dst->src[0]; const int32_t precision = KQV->op_params[2]; GGML_ASSERT(precision == GGML_PREC_DEFAULT); - constexpr int cols_per_block = 1; + constexpr int cols_per_block = 1; constexpr int parallel_blocks = 4; switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 256: - launch_fattn_vec_f16<256, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; + case 64: { + constexpr int D = 64; + constexpr int nwarps = D/WARP_SIZE; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + case 128: { + constexpr int D = 128; + constexpr int nwarps = D/WARP_SIZE; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + case 256: { + constexpr int D = 256; + constexpr int nwarps = D/WARP_SIZE; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; default: GGML_ASSERT(false); break; } } +template +void launch_fattn_vec_f16_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + switch (Q->ne[0]) { + case 64: { + constexpr int D = 64; + constexpr int nwarps = D/WARP_SIZE; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + case 128: { + constexpr int D = 128; + constexpr int nwarps = D/WARP_SIZE; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + default: { + GGML_ASSERT(false && "FlashAttention without tensor cores only supports head sizes 64 and 128."); + } break; + } +} + void ggml_cuda_flash_attn_ext_vec_f16_no_mma(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * Q = dst->src[0]; - const ggml_tensor * K = dst->src[1]; - const ggml_tensor * V = dst->src[2]; - - const ggml_tensor * mask = dst->src[3]; - - ggml_tensor * KQV = dst; + const ggml_tensor * KQV = dst; + const ggml_tensor * Q = dst->src[0]; const int32_t precision = KQV->op_params[2]; GGML_ASSERT(precision == GGML_PREC_DEFAULT); - GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); if (Q->ne[1] == 1) { - constexpr int cols_per_block = 1; - constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + ggml_cuda_flash_attn_ext_vec_f16(ctx, dst); return; } if (Q->ne[1] == 2) { - constexpr int cols_per_block = 2; + constexpr int cols_per_block = 2; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f16_64_128(ctx, dst); return; } if (Q->ne[1] <= 4) { - constexpr int cols_per_block = 4; + constexpr int cols_per_block = 4; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f16_64_128(ctx, dst); return; } if (Q->ne[1] <= 8) { - constexpr int cols_per_block = 8; + constexpr int cols_per_block = 8; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f16_64_128(ctx, dst); return; } - constexpr int cols_per_block = 8; + constexpr int cols_per_block = 8; constexpr int parallel_blocks = 1; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f16< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f16<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f16_64_128(ctx, dst); } diff --git a/ggml-cuda/fattn-vec-f32.cu b/ggml-cuda/fattn-vec-f32.cu index 91fcdc8c3..5bcabd092 100644 --- a/ggml-cuda/fattn-vec-f32.cu +++ b/ggml-cuda/fattn-vec-f32.cu @@ -52,17 +52,7 @@ static __global__ void flash_attn_vec_ext_f32( const int stride_KV = nb11 / sizeof(half); const int stride_KV2 = nb11 / sizeof(half2); - float slope = 1.0f; - - // ALiBi - if (max_bias > 0.0f) { - const uint32_t h = blockIdx.y; - - const float base = h < n_head_log2 ? m0 : m1; - const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; - - slope = powf(base, exph); - } + const float slope = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); static_assert(D % (2*WARP_SIZE) == 0, "D not divisible by 2*WARP_SIZE == 64."); constexpr int nwarps = D / WARP_SIZE; @@ -221,161 +211,65 @@ static __global__ void flash_attn_vec_ext_f32( dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; } - if (parallel_blocks != 1 && threadIdx.x < ncols) { - dst_meta[(ic0 + threadIdx.x)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[threadIdx.x], kqsum[threadIdx.x]); + if (parallel_blocks != 1 && tid < ncols) { + dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); } } -template void launch_fattn_vec_f32( - const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, - ggml_cuda_pool & pool, cudaStream_t main_stream -) { - ggml_cuda_pool_alloc dst_tmp(pool); - ggml_cuda_pool_alloc dst_tmp_meta(pool); - - if (parallel_blocks > 1) { - dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); - dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); +template +void launch_fattn_vec_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + switch (Q->ne[0]) { + case 64: { + constexpr int D = 64; + constexpr int nwarps = D/WARP_SIZE; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + case 128: { + constexpr int D = 128; + constexpr int nwarps = D/WARP_SIZE; + fattn_kernel_t fattn_kernel = flash_attn_vec_ext_f32; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); + } break; + default: { + GGML_ASSERT(false && "FlashAttention without tensor cores only supports head sizes 64 and 128."); + } break; } - - constexpr int nwarps = (D + WARP_SIZE - 1) / WARP_SIZE; - const dim3 block_dim(WARP_SIZE, nwarps, 1); - const dim3 blocks_num(parallel_blocks*((Q->ne[1] + cols_per_block - 1) / cols_per_block), Q->ne[2], Q->ne[3]); - const int shmem = 0; - - float scale = 1.0f; - float max_bias = 0.0f; - - memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); - - const uint32_t n_head = Q->ne[2]; - const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); - - const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - - flash_attn_vec_ext_f32 - <<>> ( - (const char *) Q->data, - (const char *) K->data, - (const char *) V->data, - mask ? ((const char *) mask->data) : nullptr, - parallel_blocks == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, - scale, max_bias, m0, m1, n_head_log2, - Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], - K->ne[0], K->ne[1], K->ne[2], K->ne[3], - mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, - Q->nb[1], Q->nb[2], Q->nb[3], - K->nb[1], K->nb[2], K->nb[3], - KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] - ); - CUDA_CHECK(cudaGetLastError()); - - if (parallel_blocks == 1) { - return; - } - - const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); - const int shmem_combine = 0; - - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); - CUDA_CHECK(cudaGetLastError()); } void ggml_cuda_flash_attn_ext_vec_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * Q = dst->src[0]; - const ggml_tensor * K = dst->src[1]; - const ggml_tensor * V = dst->src[2]; - - const ggml_tensor * mask = dst->src[3]; - - ggml_tensor * KQV = dst; - - GGML_ASSERT(Q->ne[0] == 64 || Q->ne[0] == 128 && "FlashAttention without tensor cores only supports head sizes 64 and 128."); if (Q->ne[1] == 1) { - constexpr int cols_per_block = 1; + constexpr int cols_per_block = 1; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f32_64_128(ctx, dst); return; } if (Q->ne[1] == 2) { - constexpr int cols_per_block = 2; + constexpr int cols_per_block = 2; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f32_64_128(ctx, dst); return; } if (Q->ne[1] <= 4) { - constexpr int cols_per_block = 4; + constexpr int cols_per_block = 4; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f32_64_128(ctx, dst); return; } if (Q->ne[1] <= 8) { - constexpr int cols_per_block = 8; + constexpr int cols_per_block = 8; constexpr int parallel_blocks = 4; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f32_64_128(ctx, dst); return; } - constexpr int cols_per_block = 8; + constexpr int cols_per_block = 8; constexpr int parallel_blocks = 1; - switch (Q->ne[0]) { - case 64: - launch_fattn_vec_f32< 64, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - case 128: - launch_fattn_vec_f32<128, cols_per_block, parallel_blocks>(Q, K, V, KQV, mask, ctx.pool(), ctx.stream()); - break; - default: - GGML_ASSERT(false); - break; - } + launch_fattn_vec_f32_64_128(ctx, dst); } diff --git a/ggml-cuda/fattn.cu b/ggml-cuda/fattn.cu index a1918e258..af7c95232 100644 --- a/ggml-cuda/fattn.cu +++ b/ggml-cuda/fattn.cu @@ -85,19 +85,9 @@ static __global__ void flash_attn_ext_f16( const int stride_Q = nb01 / sizeof(float); const int stride_KV = nb11 / sizeof(half); - half slopeh = __float2half(1.0f); - half2 slope2 = make_half2(1.0f, 1.0f); - - // ALiBi - if (max_bias > 0.0f) { - const uint32_t h = blockIdx.y; - - const float base = h < n_head_log2 ? m0 : m1; - const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; - - slopeh = __float2half(powf(base, exph)); - slope2 = make_half2(slopeh, slopeh); - } + const float slopef = get_alibi_slope(max_bias, blockIdx.y, n_head_log2, m0, m1); + const half slopeh = __float2half(slopef); + const half2 slope2 = make_half2(slopef, slopef); frag_b Q_b[D/16][ncols/frag_n]; @@ -439,108 +429,37 @@ static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed."); static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed."); static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed."); -template void launch_fattn_f16_impl( - const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, - ggml_cuda_pool & pool, cudaStream_t main_stream -) { - ggml_cuda_pool_alloc dst_tmp(pool); - ggml_cuda_pool_alloc dst_tmp_meta(pool); +template +void launch_fattn_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; - if (parallel_blocks > 1) { - dst_tmp.alloc(parallel_blocks*ggml_nelements(KQV)); - dst_tmp_meta.alloc(parallel_blocks*ggml_nrows(KQV)); - } - - constexpr int frag_m = (cols_per_block) == 8 && (D) % 32 == 0 ? 32 : 16; - const dim3 block_dim(WARP_SIZE, nwarps, 1); - const dim3 blocks_num(parallel_blocks*(Q->ne[1] + cols_per_block - 1) / cols_per_block, Q->ne[2], Q->ne[3]); - const int shmem = 0; - - float scale = 1.0f; - float max_bias = 0.0f; - - memcpy(&scale, (float *) KQV->op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) KQV->op_params + 1, sizeof(float)); - - const uint32_t n_head = Q->ne[2]; - const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head)); - - const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - - flash_attn_ext_f16 - <<>> ( - (const char *) Q->data, - (const char *) K->data, - (const char *) V->data, - mask ? ((const char *) mask->data) : nullptr, - (parallel_blocks) == 1 ? (float *) KQV->data : dst_tmp.ptr, dst_tmp_meta.ptr, - scale, max_bias, m0, m1, n_head_log2, - Q->ne[0], Q->ne[1], Q->ne[2], Q->ne[3], - K->ne[0], K->ne[1], K->ne[2], K->ne[3], - mask ? mask->ne[1] : 0, mask ? mask->nb[1] : 0, - Q->nb[1], Q->nb[2], Q->nb[3], - K->nb[1], K->nb[2], K->nb[3], - KQV->ne[0], KQV->ne[1], KQV->ne[2], KQV->ne[3] - ); - CUDA_CHECK(cudaGetLastError()); - - if ((parallel_blocks) == 1) { - return; - } - - const dim3 block_dim_combine(D, 1, 1); - const dim3 blocks_num_combine(Q->ne[1], blocks_num.y, blocks_num.z); - const int shmem_combine = 0; - - flash_attn_combine_results - <<>> - (dst_tmp.ptr, dst_tmp_meta.ptr, (float *) KQV->data); - CUDA_CHECK(cudaGetLastError()); -} - -template void launch_fattn_f16( - const ggml_tensor * Q, const ggml_tensor * K, const ggml_tensor * V, ggml_tensor * KQV, const ggml_tensor * mask, - const int nsm, ggml_cuda_pool & pool, cudaStream_t main_stream -) { + constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16; const int blocks_num_pb1 = ((Q->ne[1] + cols_per_block - 1) / cols_per_block)*Q->ne[2]*Q->ne[3]; + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; if (4*blocks_num_pb1 < 2*nsm) { - launch_fattn_f16_impl(Q, K, V, KQV, mask, pool, main_stream); + constexpr int parallel_blocks = 4; + fattn_kernel_t fattn_kernel = flash_attn_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); return; } if (2*blocks_num_pb1 < 2*nsm) { - launch_fattn_f16_impl(Q, K, V, KQV, mask, pool, main_stream); + constexpr int parallel_blocks = 2; + fattn_kernel_t fattn_kernel = flash_attn_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); return; } - launch_fattn_f16_impl(Q, K, V, KQV, mask, pool, main_stream); + constexpr int parallel_blocks = 1; + fattn_kernel_t fattn_kernel = flash_attn_ext_f16; + launch_fattn(ctx, dst, fattn_kernel, nwarps, cols_per_block); } void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * Q = dst->src[0]; - const ggml_tensor * K = dst->src[1]; - const ggml_tensor * V = dst->src[2]; - - const ggml_tensor * mask = dst->src[3]; - - ggml_tensor * KQV = dst; - - GGML_ASSERT(Q->type == GGML_TYPE_F32); - GGML_ASSERT(K->type == GGML_TYPE_F16); - GGML_ASSERT(V->type == GGML_TYPE_F16); - GGML_ASSERT(KQV->type == GGML_TYPE_F32); - - GGML_ASSERT(!mask || mask->type == GGML_TYPE_F16); - GGML_ASSERT(!mask || mask->ne[1] >= GGML_PAD(Q->ne[1], 16) && - "the Flash-Attention CUDA kernel requires the mask to be padded to 16 and at least n_queries big"); - - GGML_ASSERT(K->ne[1] % FATTN_KQ_STRIDE == 0 && "Incorrect KV cache padding."); + const ggml_tensor * KQV = dst; + const ggml_tensor * Q = dst->src[0]; ggml_cuda_set_device(ctx.device); - - const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; - const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; - + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; const int32_t precision = KQV->op_params[2]; // On AMD the tile kernels perform poorly, use the vec kernel instead: @@ -582,22 +501,22 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst constexpr int nwarps = 4; switch (Q->ne[0]) { case 64: - launch_fattn_f16< 64, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 64, cols_per_block, nwarps, float>(ctx, dst); break; case 80: - launch_fattn_f16< 80, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 80, cols_per_block, nwarps, float>(ctx, dst); break; case 96: - launch_fattn_f16< 96, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 96, cols_per_block, nwarps, float>(ctx, dst); break; case 112: - launch_fattn_f16<112, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<112, cols_per_block, nwarps, float>(ctx, dst); break; case 128: - launch_fattn_f16<128, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<128, cols_per_block, nwarps, float>(ctx, dst); break; case 256: - launch_fattn_f16<256, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<256, cols_per_block, nwarps, float>(ctx, dst); break; default: GGML_ASSERT(false); @@ -608,22 +527,22 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst constexpr int nwarps = 4; switch (Q->ne[0]) { case 64: - launch_fattn_f16< 64, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 64, cols_per_block, nwarps, float>(ctx, dst); break; case 80: - launch_fattn_f16< 80, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 80, cols_per_block, nwarps, float>(ctx, dst); break; case 96: - launch_fattn_f16< 96, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 96, cols_per_block, nwarps, float>(ctx, dst); break; case 112: - launch_fattn_f16<112, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<112, cols_per_block, nwarps, float>(ctx, dst); break; case 128: - launch_fattn_f16<128, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<128, cols_per_block, nwarps, float>(ctx, dst); break; // case 256: - // launch_fattn_f16<256, cols_per_block, nwarps, float>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + // launch_fattn_f16<256, cols_per_block, nwarps, float>(ctx, dst); // break; default: GGML_ASSERT(false); @@ -643,16 +562,16 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst constexpr int nwarps = 4; switch (Q->ne[0]) { case 64: - launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 64, cols_per_block, nwarps, half>(ctx, dst); break; case 96: - launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 96, cols_per_block, nwarps, half>(ctx, dst); break; case 128: - launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<128, cols_per_block, nwarps, half>(ctx, dst); break; case 256: - launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<256, cols_per_block, nwarps, half>(ctx, dst); break; default: GGML_ASSERT(false); @@ -666,22 +585,22 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst constexpr int nwarps = 4; switch (Q->ne[0]) { case 64: - launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 64, cols_per_block, nwarps, half>(ctx, dst); break; case 80: - launch_fattn_f16< 80, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 80, cols_per_block, nwarps, half>(ctx, dst); break; case 96: - launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 96, cols_per_block, nwarps, half>(ctx, dst); break; case 112: - launch_fattn_f16<112, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<112, cols_per_block, nwarps, half>(ctx, dst); break; case 128: - launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<128, cols_per_block, nwarps, half>(ctx, dst); break; case 256: - launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<256, cols_per_block, nwarps, half>(ctx, dst); break; default: GGML_ASSERT(false); @@ -694,22 +613,22 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst constexpr int nwarps = 4; switch (Q->ne[0]) { case 64: - launch_fattn_f16< 64, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 64, cols_per_block, nwarps, half>(ctx, dst); break; case 80: - launch_fattn_f16< 80, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 80, cols_per_block, nwarps, half>(ctx, dst); break; case 96: - launch_fattn_f16< 96, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16< 96, cols_per_block, nwarps, half>(ctx, dst); break; case 112: - launch_fattn_f16<112, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<112, cols_per_block, nwarps, half>(ctx, dst); break; case 128: - launch_fattn_f16<128, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<128, cols_per_block, nwarps, half>(ctx, dst); break; case 256: - launch_fattn_f16<256, cols_per_block, nwarps, half>(Q, K, V, KQV, mask, nsm, ctx.pool(), ctx.stream()); + launch_fattn_f16<256, cols_per_block, nwarps, half>(ctx, dst); break; default: GGML_ASSERT(false); diff --git a/ggml-cuda/softmax.cu b/ggml-cuda/softmax.cu index ca85285a3..ce64f2f2c 100644 --- a/ggml-cuda/softmax.cu +++ b/ggml-cuda/softmax.cu @@ -1,3 +1,4 @@ +#include "common.cuh" #include "softmax.cuh" template @@ -23,17 +24,7 @@ static __global__ void soft_max_f32(const float * x, const T * mask, float * dst const int warp_id = threadIdx.x / WARP_SIZE; const int lane_id = threadIdx.x % WARP_SIZE; - float slope = 1.0f; - - // ALiBi - if (max_bias > 0.0f) { - const int h = rowx/nrows_y; // head index - - const float base = h < n_head_log2 ? m0 : m1; - const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; - - slope = powf(base, exph); - } + const float slope = get_alibi_slope(max_bias, rowx/nrows_y, n_head_log2, m0, m1); extern __shared__ float data_soft_max_f32[]; float * buf_iw = data_soft_max_f32; // shared memory buffer for inter-warp communication From 511182eabb36f6ec9776e2b3c4d7e16d93d0ac0d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 18 May 2024 13:40:39 +0300 Subject: [PATCH 066/137] android : use "ci-android" branch for CI (#7341) * android : use "ci-android" branch for CI * ggml : disable SIMD exp and silu for 32-bit ARM ggml-ci * android : do not fetch, use add_subdirectory instead * cmake : provide binary dir --- .../app/src/main/cpp/CMakeLists.txt | 18 ++++++++++-------- ggml.c | 6 +++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/examples/llama.android/app/src/main/cpp/CMakeLists.txt b/examples/llama.android/app/src/main/cpp/CMakeLists.txt index 85139329a..688a4750d 100644 --- a/examples/llama.android/app/src/main/cpp/CMakeLists.txt +++ b/examples/llama.android/app/src/main/cpp/CMakeLists.txt @@ -12,15 +12,17 @@ cmake_minimum_required(VERSION 3.22.1) # build script scope). project("llama-android") -include(FetchContent) -FetchContent_Declare( - llama - GIT_REPOSITORY https://github.com/ggerganov/llama.cpp - GIT_TAG master -) +#include(FetchContent) +#FetchContent_Declare( +# llama +# GIT_REPOSITORY https://github.com/ggerganov/llama.cpp +# GIT_TAG ci-android +#) +# +## Also provides "common" +#FetchContent_MakeAvailable(llama) -# Also provides "common" -FetchContent_MakeAvailable(llama) +add_subdirectory(../../../../../../ please-work) # Creates and names a library, sets it as either STATIC # or SHARED, and provides the relative paths to its source code. diff --git a/ggml.c b/ggml.c index 55152bce4..a04c74ddd 100644 --- a/ggml.c +++ b/ggml.c @@ -2076,7 +2076,7 @@ inline static float ggml_silu_f32(float x) { return x/(1.0f + expf(-x)); } -#if defined(__ARM_NEON) +#if defined(__ARM_NEON) && defined(__aarch64__) // adapted from arm limited optimized routine // the maximum error is 1.45358 plus 0.5 ulps @@ -2288,7 +2288,7 @@ static void ggml_vec_silu_f32(const int n, float * y, const float * x) { for (; i + 3 < n; i += 4) { _mm_storeu_ps(y + i, ggml_v_silu(_mm_loadu_ps(x + i))); } -#elif defined(__ARM_NEON) +#elif defined(__ARM_NEON) && defined(__aarch64__) for (; i + 3 < n; i += 4) { vst1q_f32(y + i, ggml_v_silu(vld1q_f32(x + i))); } @@ -2335,7 +2335,7 @@ static ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, #endif sum += (ggml_float)_mm_cvtss_f32(val); } -#elif defined(__ARM_NEON) +#elif defined(__ARM_NEON) && defined(__aarch64__) for (; i + 3 < n; i += 4) { float32x4_t val = ggml_v_expf(vsubq_f32(vld1q_f32(x + i), vdupq_n_f32(max))); From 059031b8c40e1f4ba60586842c5b1ed3ddf61842 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 18 May 2024 18:55:54 +0300 Subject: [PATCH 067/137] ci : re-enable sanitizer runs (#7358) * Revert "ci : temporary disable sanitizer builds (#6128)" This reverts commit 4f6d1337ca5a409dc74aca8c479b7c34408a69c0. * ci : trigger --- .github/workflows/build.yml | 68 ++++++++++++++++++------------------ .github/workflows/server.yml | 7 ++-- CMakeLists.txt | 2 +- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0109cc004..53e61b80f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -271,40 +271,40 @@ jobs: path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-x64.zip name: llama-bin-ubuntu-x64.zip -# ubuntu-latest-cmake-sanitizer: -# runs-on: ubuntu-latest -# -# continue-on-error: true -# -# strategy: -# matrix: -# sanitizer: [ADDRESS, THREAD, UNDEFINED] -# build_type: [Debug, Release] -# -# steps: -# - name: Clone -# id: checkout -# uses: actions/checkout@v4 -# -# - name: Dependencies -# id: depends -# run: | -# sudo apt-get update -# sudo apt-get install build-essential -# -# - name: Build -# id: cmake_build -# run: | -# mkdir build -# cd build -# cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -# cmake --build . --config ${{ matrix.build_type }} -j $(nproc) -# -# - name: Test -# id: cmake_test -# run: | -# cd build -# ctest -L main --verbose --timeout 900 + ubuntu-latest-cmake-sanitizer: + runs-on: ubuntu-latest + + continue-on-error: true + + strategy: + matrix: + sanitizer: [ADDRESS, THREAD, UNDEFINED] + build_type: [Debug, Release] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v4 + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential + + - name: Build + id: cmake_build + run: | + mkdir build + cd build + cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + cmake --build . --config ${{ matrix.build_type }} -j $(nproc) + + - name: Test + id: cmake_test + run: | + cd build + ctest -L main --verbose --timeout 900 ubuntu-latest-cmake-mpi: runs-on: ubuntu-latest diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index afac89c5b..217af67cf 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -32,13 +32,14 @@ jobs: strategy: matrix: - # TODO: temporary disabled due to linux kernel issues - #sanitizer: [ADDRESS, THREAD, UNDEFINED] - sanitizer: [UNDEFINED] + sanitizer: [ADDRESS, THREAD, UNDEFINED] build_type: [Debug] include: - build_type: Release sanitizer: "" + - build_type: Debug + sanitizer: THREAD + disabled_on_pr: true fail-fast: false # While -DLLAMA_SANITIZE_THREAD=ON is broken steps: diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e4daae03..cbeb2ee37 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.14) # for add_link_options and implicit target directories. +cmake_minimum_required(VERSION 3.14) # for add_link_options and implicit target directories. project("llama.cpp" C CXX) include(CheckIncludeFileCXX) From f5bf761747988ee1832766f7d1433739aff810da Mon Sep 17 00:00:00 2001 From: fraxy-v <65565042+fraxy-v@users.noreply.github.com> Date: Sun, 19 May 2024 01:44:42 +0300 Subject: [PATCH 068/137] Capture CUDA logging output (#7298) * logging: output capture in cuda module * fix compile error * fix: vsnprintf terminates with 0, string use not correct * post review * Update llama.cpp Co-authored-by: slaren * Update llama.cpp Co-authored-by: slaren --------- Co-authored-by: slaren --- ggml-cuda.cu | 100 +++++++++++++++++++++++++++++++++++---------------- ggml-cuda.h | 1 + llama.cpp | 4 +++ 3 files changed, 75 insertions(+), 30 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 04b6e5285..754611bf3 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -43,19 +43,59 @@ #include #include #include +#include +#include #include #include static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size"); +static void ggml_cuda_default_log_callback(enum ggml_log_level level, const char * msg, void * user_data) { + GGML_UNUSED(level); + GGML_UNUSED(user_data); + fprintf(stderr, "%s", msg); +} + +ggml_log_callback ggml_cuda_log_callback = ggml_cuda_default_log_callback; +void * ggml_cuda_log_user_data = NULL; + +GGML_API void ggml_backend_cuda_log_set_callback(ggml_log_callback log_callback, void * user_data) { + ggml_cuda_log_callback = log_callback; + ggml_cuda_log_user_data = user_data; +} + +#define GGML_CUDA_LOG_INFO(...) ggml_cuda_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__) +#define GGML_CUDA_LOG_WARN(...) ggml_cuda_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__) +#define GGML_CUDA_LOG_ERROR(...) ggml_cuda_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) + +GGML_ATTRIBUTE_FORMAT(2, 3) +static void ggml_cuda_log(enum ggml_log_level level, const char * format, ...) { + if (ggml_cuda_log_callback != NULL) { + va_list args; + va_start(args, format); + char buffer[128]; + int len = vsnprintf(buffer, 128, format, args); + if (len < 128) { + ggml_cuda_log_callback(level, buffer, ggml_cuda_log_user_data); + } else { + std::vector buffer2(len + 1); // vsnprintf adds a null terminator + va_end(args); + va_start(args, format); + vsnprintf(&buffer2[0], buffer2.size(), format, args); + ggml_cuda_log_callback(level, buffer2.data(), ggml_cuda_log_user_data); + } + va_end(args); + } +} + [[noreturn]] void ggml_cuda_error(const char * stmt, const char * func, const char * file, int line, const char * msg) { int id = -1; // in case cudaGetDevice fails cudaGetDevice(&id); - fprintf(stderr, "CUDA error: %s\n", msg); - fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line); - fprintf(stderr, " %s\n", stmt); + GGML_CUDA_LOG_ERROR("CUDA error: %s\n", msg); + GGML_CUDA_LOG_ERROR(" current device: %d, in function %s at %s:%d\n", id, func, file, line); + GGML_CUDA_LOG_ERROR(" %s\n", stmt); // abort with GGML_ASSERT to get a stack trace GGML_ASSERT(!"CUDA error"); } @@ -91,7 +131,7 @@ static ggml_cuda_device_info ggml_cuda_init() { cudaError_t err = cudaGetDeviceCount(&info.device_count); if (err != cudaSuccess) { - fprintf(stderr, "%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err)); + GGML_CUDA_LOG_ERROR("%s: failed to initialize " GGML_CUDA_NAME ": %s\n", __func__, cudaGetErrorString(err)); return info; } @@ -99,16 +139,16 @@ static ggml_cuda_device_info ggml_cuda_init() { int64_t total_vram = 0; #if defined(GGML_CUDA_FORCE_MMQ) - fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__); + GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_MMQ: yes\n", __func__); #else - fprintf(stderr, "%s: GGML_CUDA_FORCE_MMQ: no\n", __func__); + GGML_CUDA_LOG_INFO("%s: GGML_CUDA_FORCE_MMQ: no\n", __func__); #endif #if defined(CUDA_USE_TENSOR_CORES) - fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: yes\n", __func__); + GGML_CUDA_LOG_INFO("%s: CUDA_USE_TENSOR_CORES: yes\n", __func__); #else - fprintf(stderr, "%s: CUDA_USE_TENSOR_CORES: no\n", __func__); + GGML_CUDA_LOG_INFO("%s: CUDA_USE_TENSOR_CORES: no\n", __func__); #endif - fprintf(stderr, "%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, info.device_count); + GGML_CUDA_LOG_INFO("%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, info.device_count); for (int id = 0; id < info.device_count; ++id) { int device_vmm = 0; @@ -129,7 +169,7 @@ static ggml_cuda_device_info ggml_cuda_init() { cudaDeviceProp prop; CUDA_CHECK(cudaGetDeviceProperties(&prop, id)); - fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no"); + GGML_CUDA_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no"); info.default_tensor_split[id] = total_vram; total_vram += prop.totalGlobalMem; @@ -235,8 +275,8 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { *actual_size = look_ahead_size; pool_size += look_ahead_size; #ifdef DEBUG_CUDA_MALLOC - fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz, - (uint32_t)(max_size/1024/1024), (uint32_t)(pool_size/1024/1024), (uint32_t)(size/1024/1024)); + GGML_CUDA_LOG_INFO("%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, device, nnz, + (uint32_t)(max_size / 1024 / 1024), (uint32_t)(pool_size / 1024 / 1024), (uint32_t)(size / 1024 / 1024)); #endif return ptr; } @@ -250,7 +290,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool { return; } } - fprintf(stderr, "WARNING: cuda buffer pool full, increase MAX_CUDA_BUFFERS\n"); + GGML_CUDA_LOG_WARN("Cuda buffer pool full, increase MAX_CUDA_BUFFERS\n"); ggml_cuda_set_device(device); CUDA_CHECK(cudaFree(ptr)); pool_size -= size; @@ -499,7 +539,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffe void * dev_ptr; cudaError_t err = cudaMalloc(&dev_ptr, size); if (err != cudaSuccess) { - fprintf(stderr, "%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size/1024.0/1024.0, buft_ctx->device, cudaGetErrorString(err)); + GGML_CUDA_LOG_ERROR("%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size / 1024.0 / 1024.0, buft_ctx->device, cudaGetErrorString(err)); return nullptr; } @@ -1002,8 +1042,8 @@ static void * ggml_cuda_host_malloc(size_t size) { if (err != cudaSuccess) { // clear the error cudaGetLastError(); - fprintf(stderr, "%s: warning: failed to allocate %.2f MiB of pinned memory: %s\n", __func__, - size/1024.0/1024.0, cudaGetErrorString(err)); + GGML_CUDA_LOG_WARN("%s: failed to allocate %.2f MiB of pinned memory: %s\n", __func__, + size / 1024.0 / 1024.0, cudaGetErrorString(err)); return nullptr; } @@ -2246,7 +2286,7 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg break; case GGML_OP_MUL_MAT: if (dst->src[0]->ne[3] != dst->src[1]->ne[3]) { - fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, dst->name, dst->src[0]->ne[3], dst->src[1]->ne[3]); + GGML_CUDA_LOG_ERROR("%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, dst->name, dst->src[0]->ne[3], dst->src[1]->ne[3]); return false; } else { ggml_cuda_mul_mat(ctx, dst->src[0], dst->src[1], dst); @@ -2300,7 +2340,7 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg cudaError_t err = cudaGetLastError(); if (err != cudaSuccess) { - fprintf(stderr, "%s: %s failed\n", __func__, ggml_op_desc(dst)); + GGML_CUDA_LOG_ERROR("%s: %s failed\n", __func__, ggml_op_desc(dst)); CUDA_CHECK(err); } @@ -2476,7 +2516,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t if (ggml_cuda_info().devices[cuda_ctx->device].cc < CC_AMPERE) { cuda_ctx->cuda_graph->disable_due_to_gpu_arch = true; #ifndef NDEBUG - fprintf(stderr, "%s: disabling CUDA graphs due to GPU architecture\n", __func__); + GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to GPU architecture\n", __func__); #endif } } @@ -2523,14 +2563,14 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t if (node->src[0] && ggml_backend_buffer_is_cuda_split(node->src[0]->buffer)) { use_cuda_graph = false; // Split buffers are not supported by CUDA graph capture #ifndef NDEBUG - fprintf(stderr, "%s: disabling CUDA graphs due to split buffer\n", __func__); + GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to split buffer\n", __func__); #endif } if (node->op == GGML_OP_MUL_MAT_ID) { use_cuda_graph = false; // This node type is not supported by CUDA graph capture #ifndef NDEBUG - fprintf(stderr, "%s: disabling CUDA graphs due to mul_mat_id\n", __func__); + GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to mul_mat_id\n", __func__); #endif } @@ -2539,7 +2579,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t // Changes in batch size or context size can cause changes to the grid size of some kernels. use_cuda_graph = false; #ifndef NDEBUG - fprintf(stderr, "%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]); + GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]); #endif } @@ -2567,7 +2607,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t if (cuda_ctx->cuda_graph->number_consecutive_updates >= 4) { cuda_ctx->cuda_graph->disable_due_to_too_many_updates = true; #ifndef NDEBUG - fprintf(stderr, "%s: disabling CUDA graphs due to too many consecutive updates\n", __func__); + GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to too many consecutive updates\n", __func__); #endif } } @@ -2605,7 +2645,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t bool ok = ggml_cuda_compute_forward(*cuda_ctx, node); if (!ok) { - fprintf(stderr, "%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); + GGML_CUDA_LOG_ERROR("%s: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); } GGML_ASSERT(ok); } @@ -2624,7 +2664,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t use_cuda_graph = false; cuda_ctx->cuda_graph->disable_due_to_failed_graph_capture = true; #ifndef NDEBUG - fprintf(stderr, "%s: disabling CUDA graphs due to failed graph capture\n", __func__); + GGML_CUDA_LOG_WARN("%s: disabling CUDA graphs due to failed graph capture\n", __func__); #endif } else { graph_evaluated_or_captured = true; // CUDA graph has been captured @@ -2691,7 +2731,7 @@ GGML_CALL static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t cudaError_t stat = cudaGraphExecUpdate(cuda_ctx->cuda_graph->instance, cuda_ctx->cuda_graph->graph, &result_info); if (stat == cudaErrorGraphExecUpdateFailure) { #ifndef NDEBUG - fprintf(stderr, "%s: CUDA graph update failed\n", __func__); + GGML_CUDA_LOG_ERROR("%s: CUDA graph update failed\n", __func__); #endif // The pre-existing graph exec cannot be updated due to violated constraints // so instead clear error and re-instantiate @@ -2948,13 +2988,13 @@ static ggml_guid_t ggml_backend_cuda_guid() { GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device) { if (device < 0 || device >= ggml_backend_cuda_get_device_count()) { - fprintf(stderr, "%s: error: invalid device %d\n", __func__, device); + GGML_CUDA_LOG_ERROR("%s: invalid device %d\n", __func__, device); return nullptr; } ggml_backend_cuda_context * ctx = new ggml_backend_cuda_context(device); if (ctx == nullptr) { - fprintf(stderr, "%s: error: failed to allocate context\n", __func__); + GGML_CUDA_LOG_ERROR("%s: failed to allocate context\n", __func__); return nullptr; } @@ -2998,8 +3038,8 @@ GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size // clear the error cudaGetLastError(); - fprintf(stderr, "%s: warning: failed to register %.2f MiB of pinned memory: %s\n", __func__, - size/1024.0/1024.0, cudaGetErrorString(err)); + GGML_CUDA_LOG_WARN("%s: failed to register %.2f MiB of pinned memory: %s\n", __func__, + size / 1024.0 / 1024.0, cudaGetErrorString(err)); return false; } return true; diff --git a/ggml-cuda.h b/ggml-cuda.h index 5eb4af40f..d7903c666 100644 --- a/ggml-cuda.h +++ b/ggml-cuda.h @@ -38,6 +38,7 @@ GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * GGML_API GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size); GGML_API GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer); +GGML_API void ggml_backend_cuda_log_set_callback(ggml_log_callback log_callback, void * user_data); #ifdef __cplusplus } #endif diff --git a/llama.cpp b/llama.cpp index 2b91eec86..1409a05da 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1697,6 +1697,8 @@ struct llama_state { llama_state() { #ifdef GGML_USE_METAL ggml_backend_metal_log_set_callback(log_callback, log_callback_user_data); +#elif defined(GGML_USE_CUDA) + ggml_backend_cuda_log_set_callback(log_callback, log_callback_user_data); #endif } @@ -18174,6 +18176,8 @@ void llama_log_set(ggml_log_callback log_callback, void * user_data) { g_state.log_callback_user_data = user_data; #ifdef GGML_USE_METAL ggml_backend_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data); +#elif defined(GGML_USE_CUDA) + ggml_backend_cuda_log_set_callback(g_state.log_callback, g_state.log_callback_user_data); #endif } From 854d365abab7194b5013a523f72da19860c3c550 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 19 May 2024 11:01:01 +0300 Subject: [PATCH 069/137] cmake : update android comments (#7341) --- examples/llama.android/app/src/main/cpp/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/llama.android/app/src/main/cpp/CMakeLists.txt b/examples/llama.android/app/src/main/cpp/CMakeLists.txt index 688a4750d..4536974a5 100644 --- a/examples/llama.android/app/src/main/cpp/CMakeLists.txt +++ b/examples/llama.android/app/src/main/cpp/CMakeLists.txt @@ -12,17 +12,20 @@ cmake_minimum_required(VERSION 3.22.1) # build script scope). project("llama-android") +## Fetch latest llama.cpp from GitHub #include(FetchContent) #FetchContent_Declare( # llama # GIT_REPOSITORY https://github.com/ggerganov/llama.cpp -# GIT_TAG ci-android +# GIT_TAG master #) # ## Also provides "common" #FetchContent_MakeAvailable(llama) -add_subdirectory(../../../../../../ please-work) +# llama.cpp CI uses the code from the current branch +# ref: https://github.com/ggerganov/llama.cpp/pull/7341#issuecomment-2117617700 +add_subdirectory(../../../../../../ build-llama) # Creates and names a library, sets it as either STATIC # or SHARED, and provides the relative paths to its source code. From e23b974f4cf9270d05062d446f406e3ff55d9451 Mon Sep 17 00:00:00 2001 From: Brian Date: Sun, 19 May 2024 20:51:03 +1000 Subject: [PATCH 070/137] labeler.yml: Use settings from ggerganov/llama.cpp [no ci] (#7363) https://github.com/actions/labeler#using-configuration-path-input-together-with-the-actionscheckout-action Recommends the use of checkout action to use the correct repo context when applying settings for PR labels e.g. steps: - uses: actions/checkout@v4 # Uploads repository content to the runner with: repository: "owner/repositoryName" # The one of the available inputs, visit https://github.com/actions/checkout#readme to find more - uses: actions/labeler@v5 with: configuration-path: 'path/to/the/uploaded/configuration/file' --- .github/workflows/labeler.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index e57cd86e2..368dbdbe5 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -9,4 +9,9 @@ jobs: pull-requests: write runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + with: + repository: "ggerganov/llama.cpp" - uses: actions/labeler@v5 + with: + configuration-path: '.github/labeler.yml' From ab33f7a338593f6cf1ae98b10b6f8684f63bd72c Mon Sep 17 00:00:00 2001 From: slaren Date: Sun, 19 May 2024 14:19:37 +0200 Subject: [PATCH 071/137] cuda : clear error after buffer allocation failure (#7376) --- ggml-cuda.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 754611bf3..b82167cbf 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -539,6 +539,8 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffe void * dev_ptr; cudaError_t err = cudaMalloc(&dev_ptr, size); if (err != cudaSuccess) { + // clear the error + cudaGetLastError(); GGML_CUDA_LOG_ERROR("%s: allocating %.2f MiB on device %d: cudaMalloc failed: %s\n", __func__, size / 1024.0 / 1024.0, buft_ctx->device, cudaGetErrorString(err)); return nullptr; } From 6aade19ee74b896c59929676629340b36be3e22c Mon Sep 17 00:00:00 2001 From: Anas Ahouzi <112881240+aahouzi@users.noreply.github.com> Date: Sun, 19 May 2024 14:46:46 +0200 Subject: [PATCH 072/137] Add StableLM2 pre-tokenizer (#7349) * Add StableLM pre-tokenizer * Fix space * Fix trailing whitespace --- convert-hf-to-gguf-update.py | 1 + convert-hf-to-gguf.py | 3 +++ llama.cpp | 4 ++++ llama.h | 7 ++++--- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 27983fadf..45404b32b 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -72,6 +72,7 @@ models = [ {"name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", }, {"name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", }, {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", }, + {"name": "stablelm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b", }, {"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", }, {"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", }, {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", }, diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index cd1750aa3..bd303150a 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -446,6 +446,9 @@ class Model: if chkhsh == "3ce83efda5659b07b1ad37ca97ca5797ea4285d9b9ab0dc679e4a720c9da7454": # ref: https://huggingface.co/openai-community/gpt2 res = "gpt-2" + if chkhsh == "32d85c31273f8019248f2559fed492d929ea28b17e51d81d3bb36fff23ca72b3": + # ref: https://huggingface.co/stabilityai/stablelm-2-1_6b + res = "stablelm2" if chkhsh == "6221ad2852e85ce96f791f476e0b390cf9b474c9e3d1362f53a24a06dc8220ff": # ref: https://huggingface.co/smallcloudai/Refact-1_6-base res = "refact" diff --git a/llama.cpp b/llama.cpp index 1409a05da..06ff4da61 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4463,6 +4463,9 @@ static void llm_load_vocab( } else if ( tokenizer_pre == "qwen2") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_QWEN2; + } else if ( + tokenizer_pre == "stablelm2") { + vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_STABLELM2; } else if ( tokenizer_pre == "olmo") { vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_OLMO; @@ -12363,6 +12366,7 @@ struct llm_tokenizer_bpe { "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)", }); break; + case LLAMA_VOCAB_PRE_TYPE_STABLELM2: case LLAMA_VOCAB_PRE_TYPE_QWEN2: word_collection = unicode_regex_split(text, { // original regex from tokenizer.json diff --git a/llama.h b/llama.h index 612e32c4e..b7bf2afcb 100644 --- a/llama.h +++ b/llama.h @@ -81,9 +81,10 @@ extern "C" { LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, LLAMA_VOCAB_PRE_TYPE_REFACT = 8, LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, - LLAMA_VOCAB_PRE_TYPE_QWEN2 = 10, - LLAMA_VOCAB_PRE_TYPE_OLMO = 11, - LLAMA_VOCAB_PRE_TYPE_DBRX = 12, + LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, + LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, + LLAMA_VOCAB_PRE_TYPE_OLMO = 12, + LLAMA_VOCAB_PRE_TYPE_DBRX = 13, }; // note: these values should be synchronized with ggml_rope From 41858392e17abead21735309bf17cb55183d8c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sun, 19 May 2024 16:06:33 +0200 Subject: [PATCH 073/137] server: fix seed being reported back (#7382) --- examples/server/server.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index b598076d3..7978f979d 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -102,7 +102,6 @@ struct slot_params { bool stream = true; bool cache_prompt = false; // remember the prompt to avoid reprocessing all prompt - uint32_t seed = -1; // RNG seed int32_t n_keep = 0; // number of tokens to keep from initial prompt int32_t n_discard = 0; // number of tokens after n_keep that may be discarded when shifting context, 0 defaults to half int32_t n_predict = -1; // new tokens to predict @@ -1264,7 +1263,7 @@ struct server_context { {"n_ctx", slot.n_ctx}, {"n_predict", slot.n_predict}, {"model", params.model_alias}, - {"seed", slot.params.seed}, + {"seed", slot.sparams.seed}, {"temperature", slot.sparams.temp}, {"dynatemp_range", slot.sparams.dynatemp_range}, {"dynatemp_exponent", slot.sparams.dynatemp_exponent}, From 1b01f06db0cff5f5f600bb754fc39fde565ed56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sun, 19 May 2024 16:26:02 +0200 Subject: [PATCH 074/137] server: add test for token probs (#7347) --- examples/server/README.md | 2 +- .../server/tests/features/results.feature | 52 ++++++++++++++++--- examples/server/tests/features/steps/steps.py | 37 ++++++++++++- 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/examples/server/README.md b/examples/server/README.md index 4f3262cdd..0c3db8c84 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -48,7 +48,7 @@ The project is under active development, and we are [looking for feedback and co - `--api-key`: Set an api key for request authorization. By default, the server responds to every request. With an api key set, the requests must have the Authorization header set with the api key as Bearer token. May be used multiple times to enable multiple valid keys. - `--api-key-file`: Path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access. May be used in conjunction with `--api-key`s. - `--embeddings`: Enable embedding vector output and the OAI compatible endpoint /v1/embeddings. Physical batch size (`--ubatch-size`) must be carefully defined. Default: disabled -- `-np N`, `--parallel N`: Set the number of slots for process requests. Default: `1` +- `-np N`, `--parallel N`: Set the number of slots for process requests. Default: `1`. Values > 1 will allow for higher throughput with multiple parallel requests but the results will **not** be deterministic due to differences in rounding error. - `-cb`, `--cont-batching`: Enable continuous batching (a.k.a dynamic batching). Default: disabled - `-spf FNAME`, `--system-prompt-file FNAME` Set a file to load a system prompt (initial prompt of all slots). This is useful for chat applications. [See more](#change-system-prompt-on-runtime) - `--mmproj MMPROJ_FILE`: Path to a multimodal projector file for LLaVA. diff --git a/examples/server/tests/features/results.feature b/examples/server/tests/features/results.feature index aa0b8d0c6..5deb278c2 100644 --- a/examples/server/tests/features/results.feature +++ b/examples/server/tests/features/results.feature @@ -70,12 +70,48 @@ Feature: Results Then all predictions are equal Examples: | n_parallel | temp | - | 1 | 0.0 | - | 2 | 0.0 | - | 4 | 0.0 | - | 1 | 1.0 | - # FIXME: These tests fail on master. The problem seems to be the unified KV cache. + | 1 | 0.0 | + | 2 | 0.0 | + | 4 | 0.0 | + | 1 | 1.0 | + # FIXME: These tests fail on master. + # Problems: unified KV cache (except for CPU backend with LLAMA_NO_LLAMAFILE=1), SIMD nondeterminism. # See https://github.com/ggerganov/whisper.cpp/issues/1941#issuecomment-1986923227 - # and https://github.com/ggerganov/llama.cpp/pull/6122#discussion_r1531405574 . - # | 2 | 1.0 | - # | 4 | 1.0 | + # and https://github.com/ggerganov/llama.cpp/pull/6122#discussion_r1531405574 + # and https://github.com/ggerganov/llama.cpp/pull/7347 . + # | 2 | 1.0 | + # | 4 | 1.0 | + + Scenario Outline: consistent token probs with same seed and prompt + Given slots + And KV cache size + And 1.0 temperature + And max tokens to predict + Then the server is starting + Then the server is healthy + + Given 1 prompts "The meaning of life is" with seed 42 + And concurrent completion requests + # Then the server is busy # Not all slots will be utilized. + Then the server is idle + And all slots are idle + + Given prompts "The meaning of life is" with seed 42 + And concurrent completion requests + # Then the server is busy # Not all slots will be utilized. + Then the server is idle + And all slots are idle + + Then all token probabilities are equal + Examples: + | n_slots | n_kv | n_predict | n_parallel | + | 4 | 1024 | 1 | 1 | + | 4 | 1024 | 1 | 4 | + # FIXME: These tests fail on master. + # Problems: unified KV cache (except for CPU backend with LLAMA_NO_LLAMAFILE=1), SIMD nondeterminism. + # See https://github.com/ggerganov/whisper.cpp/issues/1941#issuecomment-1986923227 + # and https://github.com/ggerganov/llama.cpp/pull/6122#discussion_r1531405574 + # and https://github.com/ggerganov/llama.cpp/pull/7347 . + # | 4 | 1024 | 100 | 1 | + # This test still fails even the above patches; the first token probabilities are already different. + # | 4 | 1024 | 100 | 4 | diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 577b87af3..7da503f2c 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -23,6 +23,7 @@ from prometheus_client import parser def step_server_config(context, server_fqdn, server_port): context.server_fqdn = server_fqdn context.server_port = int(server_port) + context.n_threads = None context.n_gpu_layer = None if 'PORT' in os.environ: context.server_port = int(os.environ['PORT']) @@ -109,6 +110,11 @@ def step_n_gpu_layer(context, ngl): context.n_gpu_layer = ngl +@step('{n_threads:d} threads') +def step_n_threads(context, n_threads): + context.n_thread = n_threads + + @step('{draft:d} as draft') def step_draft(context, draft): context.draft = draft @@ -274,13 +280,22 @@ async def step_predictions_equal(context): @step('all predictions are different') @async_run_until_complete -async def step_predictions_equal(context): +async def step_predictions_different(context): n_completions = await gather_tasks_results(context) assert n_completions >= 2, "need at least 2 completions" assert_all_predictions_different(context.tasks_result) context.tasks_result = [] +@step('all token probabilities are equal') +@async_run_until_complete +async def step_token_probabilities_equal(context): + n_completions = await gather_tasks_results(context) + assert n_completions >= 2, "need at least 2 completions" + assert_all_token_probabilities_equal(context.tasks_result) + context.tasks_result = [] + + @step('the completion is truncated') def step_assert_completion_truncated(context): step_assert_completion_truncated(context, '') @@ -869,6 +884,7 @@ async def request_completion(prompt, "id_slot": id_slot, "seed": seed if seed is not None else 42, "temperature": temperature if temperature is not None else "0.8f", + "n_probs": 2, }, headers=headers, timeout=3600) as response: @@ -1123,6 +1139,23 @@ def assert_all_predictions_different(completion_responses): assert content_i != content_j, "contents not different" +def assert_all_token_probabilities_equal(completion_responses): + n_predict = len(completion_responses[0]['completion_probabilities']) + if 'DEBUG' in os.environ and os.environ['DEBUG'] == 'ON': + for pos in range(n_predict): + for i, response_i in enumerate(completion_responses): + probs_i = response_i['completion_probabilities'][pos]['probs'] + print(f"pos {pos}, probs {i}: {probs_i}") + for pos in range(n_predict): + for i, response_i in enumerate(completion_responses): + probs_i = response_i['completion_probabilities'][pos]['probs'] + for j, response_j in enumerate(completion_responses): + if i == j: + continue + probs_j = response_j['completion_probabilities'][pos]['probs'] + assert probs_i == probs_j, "contents not equal" + + async def gather_tasks_results(context): n_tasks = len(context.concurrent_tasks) if context.debug: @@ -1261,6 +1294,8 @@ def start_server_background(context): server_args.extend(['--batch-size', context.n_batch]) if context.n_ubatch: server_args.extend(['--ubatch-size', context.n_ubatch]) + if context.n_threads: + server_args.extend(['--threads', context.threads]) if context.n_gpu_layer: server_args.extend(['--n-gpu-layers', context.n_gpu_layer]) if context.draft is not None: From 5ca49cbecda27ce0a7266658fc3b640bff3ed386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Sun, 19 May 2024 16:46:13 +0200 Subject: [PATCH 075/137] ggml: implement quantized KV cache for FA (#7372) --- ggml.c | 115 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 43 deletions(-) diff --git a/ggml.c b/ggml.c index a04c74ddd..3a104c486 100644 --- a/ggml.c +++ b/ggml.c @@ -15882,9 +15882,10 @@ static void ggml_compute_forward_flash_attn_ext_f16( GGML_ASSERT(ne0 == D); GGML_ASSERT(ne2 == N); - GGML_ASSERT(nbq0 == sizeof(float)); - GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t)); + // input tensor rows must be contiguous + GGML_ASSERT(nbq0 == ggml_type_size(q->type)); + GGML_ASSERT(nbk0 == ggml_type_size(k->type)); + GGML_ASSERT(nbv0 == ggml_type_size(v->type)); GGML_ASSERT(neq0 == D); GGML_ASSERT(nek0 == D); @@ -15938,6 +15939,11 @@ static void ggml_compute_forward_flash_attn_ext_f16( const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); + enum ggml_type const k_vec_dot_type = type_traits[k->type].vec_dot_type; + ggml_from_float_t const q_to_vec_dot = type_traits[k_vec_dot_type].from_float; + ggml_vec_dot_t const kq_vec_dot = type_traits[k->type].vec_dot; + ggml_to_float_t const v_to_float = type_traits[v->type].to_float; + // loop over n_batch and n_head for (int ir = ir0; ir < ir1; ++ir) { // q indices @@ -15945,17 +15951,22 @@ static void ggml_compute_forward_flash_attn_ext_f16( const int iq2 = (ir - iq3*neq2*neq1)/neq1; const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1); - const uint32_t h = iq2; // head + const uint32_t h = iq2; // head index const float slope = (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2*(h - n_head_log2) + 1) : 1.0f; - float S = 0.0f; - float M = -INFINITY; + float S = 0.0f; // sum + float M = -INFINITY; // maximum KQ value - float * V32 = (float *) params->wdata + ith*(2*D + CACHE_LINE_SIZE_F32); - ggml_fp16_t * Q16 = (ggml_fp16_t *) (V32); // reuse memory - ggml_fp16_t * V16 = (ggml_fp16_t *) (V32 + D); + float * VKQ32 = (float *) params->wdata + ith*(3*D + CACHE_LINE_SIZE_F32); // FP32 VKQ accumulator + float * V32 = (VKQ32 + 1*D); // (temporary) FP32 V buffer + ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*D); // (temporary) FP16 VKQ accumulator + ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*D); // (temporary) buffer for Q converted to quantized/FP16 - memset(V16, 0, D*sizeof(ggml_fp16_t)); + if (v->type == GGML_TYPE_F16) { + memset(VKQ16, 0, D*sizeof(ggml_fp16_t)); + } else { + memset(VKQ32, 0, D*sizeof(float)); + } const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1]) : NULL; @@ -15967,6 +15978,9 @@ static void ggml_compute_forward_flash_attn_ext_f16( const int iv3 = iq3 / rv3; const int iv2 = iq2 / rv2; + const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)); + q_to_vec_dot(pq, Q_q, D); + // online softmax / attention // loop over n_kv and n_head_kv // ref: https://arxiv.org/pdf/2112.05682.pdf @@ -15976,51 +15990,66 @@ static void ggml_compute_forward_flash_attn_ext_f16( continue; } - float s; + float s; // KQ value - // convert Q to F16 in V32 - { - const float * pq = (const float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)); + const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3); + kq_vec_dot(D, &s, 0, k_data, 0, Q_q, 0, 1); - for (int64_t d = 0; d < D; ++d) { - Q16[d] = GGML_FP32_TO_FP16(pq[d]); - } - } - - ggml_vec_dot_f16(D, - &s, 0, - (ggml_fp16_t *) ((char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3)), 0, - Q16, 0, 1); - - s = s*scale + mv; + s = s*scale + mv; // scale KQ value and apply mask const float Mold = M; - float ms = 1.0f; - float vs = 1.0f; + float ms = 1.0f; // upon new higher max val, scale VKQ and KQ sum with this value + float vs = 1.0f; // post-softmax KQ value, expf(s - M) - if (s > M) { - M = s; - ms = expf(Mold - M); + const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3)); - // V = V*expf(Mold - M) - ggml_vec_scale_f16(D, V16, ms); + if (v->type== GGML_TYPE_F16) { + if (s > M) { + // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f + M = s; + ms = expf(Mold - M); + + // V = V*expf(Mold - M) + ggml_vec_scale_f16(D, VKQ16, ms); + } else { + // no new maximum, ms == 1.0f, vs != 1.0f + vs = expf(s - M); + } + + // V += v*expf(s - M) + ggml_vec_mad_f16(D, VKQ16, (const ggml_fp16_t *) v_data, vs); } else { - vs = expf(s - M); + if (s > M) { + // s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f + M = s; + ms = expf(Mold - M); + + // V = V*expf(Mold - M) + ggml_vec_scale_f32(D, VKQ32, ms); + } else { + // no new maximum, ms == 1.0f, vs != 1.0f + vs = expf(s - M); + } + + v_to_float(v_data, V32, D); + + // V += v*expf(s - M) + ggml_vec_mad_f32(D, VKQ32, V32, vs); } - const ggml_fp16_t * v16 = (const ggml_fp16_t *) ((char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3)); + S = S*ms + vs; // scale and increment sum with partial sum + } - // V += v*expf(s - M) - ggml_vec_mad_f16(D, V16, v16, vs); - - S = S*ms + vs; + if (v->type == GGML_TYPE_F16) { + for (int64_t d = 0; d < D; ++d) { + VKQ32[d] = GGML_FP16_TO_FP32(VKQ16[d]); + } } // V /= S - for (int64_t d = 0; d < D; ++d) { - V32[d] = GGML_FP16_TO_FP32(V16[d])/S; - } + const float S_inv = 1.0f/S; + ggml_vec_scale_f32(D, VKQ32, S_inv); // dst indices const int i1 = iq1; @@ -16031,7 +16060,7 @@ static void ggml_compute_forward_flash_attn_ext_f16( //memcpy((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3), V, nev0*sizeof(float)); // permute(0, 2, 1, 3) - memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, V32, nb1); + memcpy((char *) dst->data + (i3*ne2*ne1 + i2 + i1*ne1)*nb1, VKQ32, nb1); } } @@ -19972,7 +20001,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa { const int64_t ne00 = node->src[0]->ne[0]; // D - cur = 2*sizeof(float)*ne00*n_tasks; // 2x head size + cur = 3*sizeof(float)*ne00*n_tasks; // 3x head size/thread } break; case GGML_OP_FLASH_FF: { From e4e6f67be6a8a697f5f89a28c98934e53c99c359 Mon Sep 17 00:00:00 2001 From: slaren Date: Sun, 19 May 2024 17:08:46 +0200 Subject: [PATCH 076/137] ggml : fix another case of quants nans (#7387) --- ggml-quants.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-quants.c b/ggml-quants.c index 7008e5d8b..ff1059863 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1149,7 +1149,7 @@ static float make_qx_quants(int n, int nmax, const float * restrict x, int8_t * sumlx += w*x[i]*l; suml2 += w*l*l; } - float scale = sumlx/suml2; + float scale = suml2 ? sumlx/suml2 : 0.0f; if (return_early) return suml2 > 0 ? 0.5f*(scale + 1/iscale) : 1/iscale; float best = scale * sumlx; for (int is = -9; is <= 9; ++is) { From f030ec1f7a72aa825b2104823946551b9ec5dfc1 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Sun, 19 May 2024 17:19:53 +0200 Subject: [PATCH 077/137] Vulkan Embedding Fix (#7360) * Fix empty Vulkan host buffers Add fp32 fp16 matmul shader Fix matmul shader alignment * Remove deprecated tensor->backend uses * Fix Vulkan validation errors on embedding models with no offloaded layers * Fix Vulkan llava segfault when not offloading layers --- ggml-vulkan-shaders.hpp | 12204 ++++++++++++++++++++++------------ ggml-vulkan.cpp | 184 +- ggml_vk_generate_shaders.py | 9 +- 3 files changed, 8000 insertions(+), 4397 deletions(-) diff --git a/ggml-vulkan-shaders.hpp b/ggml-vulkan-shaders.hpp index 586bafe46..70c4043d3 100644 --- a/ggml-vulkan-shaders.hpp +++ b/ggml-vulkan-shaders.hpp @@ -12023,7 +12023,7 @@ const uint64_t matmul_f16_len = 10316; unsigned char matmul_f16_aligned_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xf5,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x08,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -12031,9 +12031,9 @@ unsigned char matmul_f16_aligned_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x55,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x69,0x01,0x00,0x00, +0x68,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, @@ -12076,37 +12076,40 @@ unsigned char matmul_f16_aligned_data[] = { 0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x05,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x05,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x03,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3c,0x01,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4e,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x67,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x67,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4f,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x67,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x47,0x00,0x03,0x00,0x67,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x69,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x69,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x55,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9b,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x9c,0x02,0x00,0x00, +0x68,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xae,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xaf,0x02,0x00,0x00, 0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x9c,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x9c,0x02,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9e,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xaf,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb1,0x02,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9e,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0xb1,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -12157,16 +12160,16 @@ unsigned char matmul_f16_aligned_data[] = { 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, 0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, 0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x06,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, @@ -12209,727 +12212,756 @@ unsigned char matmul_f16_aligned_data[] = { 0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00, 0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x15,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xf7,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x16,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x06,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x12,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x48,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x48,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x4c,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x4d,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, -0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x85,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x78,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, -0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xaa,0x01,0x00,0x00, -0xf7,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xab,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc1,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xd8,0x01,0x00,0x00, -0xf7,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xd9,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x62,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x66,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x68,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x68,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x99,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, 0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x55,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x9b,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x9d,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, -0x3b,0x00,0x04,0x00,0x9d,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x84,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xb9,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xab,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xd9,0x01,0x00,0x00,0xda,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xc3,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xc3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x3e,0x02,0x00,0x00, -0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0xe0,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xd4,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdf,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xd4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xc2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xff,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, -0x0e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0xfc,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x16,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x17,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0xd4,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, -0x1a,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, -0x20,0x01,0x00,0x00,0xfc,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe0,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0xd4,0x02,0x00,0x00, -0x25,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x89,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2f,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0xdc,0x02,0x00,0x00,0x38,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x39,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x40,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x41,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x12,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x57,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x60,0x01,0x00,0x00, -0x5f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, -0x61,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x12,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x64,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x68,0x01,0x00,0x00, -0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0xd0,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6f,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x71,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x75,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, -0x76,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x72,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x12,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7d,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0xd0,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x01,0x00,0x00,0xd5,0x02,0x00,0x00, -0x87,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0xd8,0x02,0x00,0x00,0x8b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0xdc,0x02,0x00,0x00, -0x8e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x92,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x92,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x3c,0x02,0x00,0x00,0x95,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x98,0x01,0x00,0x00, -0xde,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x94,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x98,0x01,0x00,0x00,0x93,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x93,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0xe2,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9c,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa0,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa2,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xa8,0x01,0x00,0x00,0xf4,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa8,0x01,0x00,0x00, -0xa3,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xae,0x01,0x00,0x00,0xe2,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0xae,0x01,0x00,0x00,0xf4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x01,0x00,0x00,0xe2,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb5,0x01,0x00,0x00, -0xb2,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0x65,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0xb8,0x01,0x00,0x00,0xf4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0xde,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc1,0x01,0x00,0x00,0xc2,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc2,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc4,0x01,0x00,0x00,0xf4,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa2,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, -0xe2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xc8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xc8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xe3,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xca,0x01,0x00,0x00, -0xcb,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xce,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xc9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd0,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xd6,0x01,0x00,0x00,0xf1,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd6,0x01,0x00,0x00, -0xd1,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xdc,0x01,0x00,0x00,0xe3,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00, -0xdc,0x01,0x00,0x00,0xf1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe3,0x02,0x00,0x00,0xe2,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00, -0xe0,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe6,0x01,0x00,0x00,0x69,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0xf1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, -0xea,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xed,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0xde,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xee,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xc1,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xf1,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xcb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xcb,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, -0xe3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xc8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xca,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xca,0x01,0x00,0x00, -0x3a,0x02,0x00,0x00,0xf9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xe4,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf8,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xfc,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0xf8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xfe,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0x38,0x02,0x00,0x00, -0x01,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x04,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x00,0x02,0x00,0x00,0x01,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x04,0x02,0x00,0x00, -0xff,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x06,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x06,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0x36,0x02,0x00,0x00,0x09,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x0c,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x08,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0c,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xec,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0xec,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x10,0x02,0x00,0x00, -0x0f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x14,0x02,0x00,0x00,0x0f,0x02,0x00,0x00,0x10,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, -0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xea,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0x1a,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, -0x1d,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, -0xec,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xc1,0x01,0x00,0x00, -0x24,0x02,0x00,0x00,0xac,0x01,0x00,0x00,0x23,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x25,0x02,0x00,0x00, -0x24,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xc1,0x01,0x00,0x00,0x2b,0x02,0x00,0x00,0xda,0x01,0x00,0x00, -0x18,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x2c,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0xca,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x31,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x2f,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0xec,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x10,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, -0xea,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x06,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x01,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x01,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xfe,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xf9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0xe4,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x95,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x95,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, -0xde,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x92,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x94,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0xbd,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xbc,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xd4,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xea,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xeb,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xea,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xec,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xeb,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x68,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xae,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xaf,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xb0,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xb0,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xd6,0x02,0x00,0x00,0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xd6,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xef,0x02,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xeb,0x02,0x00,0x00,0x9f,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00,0x85,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0xd7,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xd7,0x02,0x00,0x00,0x8f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, +0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x17,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x1a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x25,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x2c,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x33,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x36,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0xe7,0x02,0x00,0x00,0x3f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x43,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x9c,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xe8,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x45,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x49,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0xe8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0xef,0x02,0x00,0x00, +0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x69,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x79,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x7e,0x01,0x00,0x00,0x7d,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x84,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x85,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x69,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x8b,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x2e,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x95,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, +0xe8,0x02,0x00,0x00,0x9a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x45,0x01,0x00,0x00, 0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3e,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x56,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x02,0x00,0x00,0x97,0x00,0x00,0x00,0x43,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x02,0x00,0x00, -0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0xa8,0x00,0x00,0x00, -0x49,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, -0x4e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x50,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, -0x48,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x55,0x02,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x02,0x00,0x00,0x50,0x02,0x00,0x00,0x58,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5d,0x02,0x00,0x00, -0x5e,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x61,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x69,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x69,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x64,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, -0x44,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x65,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0xeb,0x02,0x00,0x00,0x9e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00, +0xef,0x02,0x00,0x00,0xa1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x4f,0x02,0x00,0x00, +0xa8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xab,0x01,0x00,0x00,0xf1,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xa7,0x01,0x00,0x00,0xa8,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xab,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0xf5,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0xaf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x07,0x03,0x00,0x00,0x3f,0x00,0x00,0x00,0xae,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x07,0x03,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xf5,0x02,0x00,0x00, 0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x71,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0x70,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, -0xc5,0x02,0x00,0x00,0xe2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x4a,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0xbe,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x81,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x86,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0xca,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x85,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x89,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x91,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8f,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x90,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x94,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0x94,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x91,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00,0x98,0x02,0x00,0x00, -0x8f,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x97,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x9a,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x98,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x99,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x59,0x02,0x00,0x00, -0xa6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa9,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0x71,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x07,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00, +0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xf5,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00, +0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcd,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0x07,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, +0xcd,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, +0xf1,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xd4,0x01,0x00,0x00, +0xd5,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xd5,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0x07,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd9,0x01,0x00,0x00,0xf5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xaf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xaf,0x01,0x00,0x00,0x07,0x02,0x00,0x00,0xde,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xf6,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe1,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x04,0x03,0x00,0x00,0x3f,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0x05,0x02,0x00,0x00,0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x04,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe5,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe9,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xf6,0x02,0x00,0x00, 0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0xaf,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xb1,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, -0xb4,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0xca,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb7,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0xb9,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x35,0x00,0x00,0x00,0xab,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xba,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x86,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x85,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, -0xc6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x63,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc2,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5d,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0xf1,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x04,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x01,0x00,0x00, +0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0xf6,0x02,0x00,0x00, +0xf5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, +0xf9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfc,0x01,0x00,0x00,0xfa,0x01,0x00,0x00,0x04,0x03,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0xf1,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x63,0x01,0x00,0x00,0x00,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x01,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xd4,0x01,0x00,0x00, +0x03,0x02,0x00,0x00,0xed,0x01,0x00,0x00,0xf1,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x03,0x02,0x00,0x00,0x02,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x04,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0xf6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x09,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xdd,0x01,0x00,0x00,0x4d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x0f,0x02,0x00,0x00, +0xf7,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x0b,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x0f,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, +0x0b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x11,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x11,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfb,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x4b,0x02,0x00,0x00,0x14,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0xfb,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x13,0x02,0x00,0x00, +0x14,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x17,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x13,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfd,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x49,0x02,0x00,0x00, +0x1c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xfd,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x1b,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1f,0x02,0x00,0x00, +0x1a,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x21,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x21,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xff,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x22,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0xff,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x23,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x27,0x02,0x00,0x00,0x22,0x02,0x00,0x00, +0x23,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00, +0xf7,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x29,0x02,0x00,0x00, +0xfd,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2d,0x02,0x00,0x00,0x2b,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, +0xfb,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x30,0x02,0x00,0x00,0xff,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0xff,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xd4,0x01,0x00,0x00,0x37,0x02,0x00,0x00,0xbf,0x01,0x00,0x00, +0x36,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xd4,0x01,0x00,0x00,0x3e,0x02,0x00,0x00, +0xed,0x01,0x00,0x00,0x2b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x40,0x02,0x00,0x00, +0x3f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x42,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x42,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x44,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x39,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x43,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x42,0x02,0x00,0x00,0x44,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, +0xff,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x21,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x02,0x00,0x00,0xfd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00,0xfb,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x11,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x13,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x0c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, +0xf7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0xf1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa7,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0xd7,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x97,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x02,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0x60,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x63,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x62,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x69,0x02,0x00,0x00, +0x68,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6b,0x02,0x00,0x00, +0x67,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x63,0x02,0x00,0x00, +0x6b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x74,0x02,0x00,0x00, +0xd8,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x74,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, +0x70,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd9,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x6f,0x02,0x00,0x00, +0xd3,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x7c,0x02,0x00,0x00,0xd9,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x78,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7c,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0xd9,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x81,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x02,0x00,0x00, +0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x88,0x02,0x00,0x00,0xd8,0x02,0x00,0x00,0xf5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x69,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x8e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x8e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xdb,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0xd1,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0xdb,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x90,0x02,0x00,0x00, +0x91,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x94,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x90,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x96,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x96,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0xcf,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0xdd,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x98,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x9c,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x97,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0xa4,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa2,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0xdb,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xaa,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa4,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0x97,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0xad,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xab,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xac,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x8c,0x02,0x00,0x00, +0xdb,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xb7,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0xb6,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0xb7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00, +0x6c,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xba,0x02,0x00,0x00, +0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbe,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0xd8,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, +0xdb,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0xc6,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xca,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0xca,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0xcc,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0xb1,0x02,0x00,0x00,0x35,0x00,0x00,0x00,0xbe,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x99,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xdd,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x96,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x98,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x91,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x91,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, +0xdb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x8e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x90,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd3,0x02,0x00,0x00,0xd9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x71,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00,0xd8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x70,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t matmul_f16_aligned_len = 10812; +const uint64_t matmul_f16_aligned_len = 11200; unsigned char matmul_f16_aligned_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xd9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0xd8,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, @@ -12937,8 +12969,8 @@ unsigned char matmul_f16_aligned_fp32_data[] = { 0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x14,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x39,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0xfd,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x38,0x02,0x00,0x00,0x81,0x02,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, @@ -12981,17 +13013,17 @@ unsigned char matmul_f16_aligned_fp32_data[] = { 0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x02,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x21,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x22,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x4d,0x01,0x00,0x00,0x06,0x00,0x00,0x00, 0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4e,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, @@ -13000,15 +13032,15 @@ unsigned char matmul_f16_aligned_fp32_data[] = { 0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x01,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x50,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x39,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, +0x47,0x00,0x04,0x00,0x38,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, 0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x80,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x7f,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x7f,0x02,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x80,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x82,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x82,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x7f,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x81,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x21,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, 0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, @@ -13059,16 +13091,16 @@ unsigned char matmul_f16_aligned_fp32_data[] = { 0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, 0x68,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00, 0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, 0x87,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, @@ -13112,99 +13144,97 @@ unsigned char matmul_f16_aligned_fp32_data[] = { 0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, 0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x03,0x01,0x00,0x00, -0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x12,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x48,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x48,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x4c,0x01,0x00,0x00,0x03,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4d,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x4e,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x4f,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, -0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6c,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6e,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x71,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xff,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x00,0x00, 0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x90,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x48,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x49,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4d,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x4f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, +0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x84,0x00,0x00,0x00, 0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x91,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x90,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x92,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x90,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x91,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x90,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, 0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, -0xc4,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0xbf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc8,0x01,0x00,0x00, +0xbc,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xbd,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xbe,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, 0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00, 0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x30,0x02,0x00,0x00, 0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x39,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x7f,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x80,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x20,0x00,0x04,0x00, -0x81,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x3b,0x00,0x04,0x00,0x81,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x7e,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x7f,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0x80,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x7f,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0x80,0x02,0x00,0x00,0x81,0x02,0x00,0x00, 0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x87,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x84,0x00,0x00,0x00, 0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x9d,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, 0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, 0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x92,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbe,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, 0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, 0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, @@ -13261,9 +13291,9 @@ unsigned char matmul_f16_aligned_fp32_data[] = { 0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x64,0x00,0x00,0x00, 0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, 0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x73,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x74,0x00,0x00,0x00, 0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, 0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, 0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, @@ -13296,7 +13326,7 @@ unsigned char matmul_f16_aligned_fp32_data[] = { 0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, 0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00, 0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x9e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, 0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00, 0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, 0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, @@ -13312,479 +13342,481 @@ unsigned char matmul_f16_aligned_fp32_data[] = { 0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, 0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00, 0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xaf,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, 0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, 0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, 0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, 0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, 0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, 0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, 0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, 0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, 0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, 0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0xbf,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x76,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x21,0x02,0x00,0x00, 0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, 0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, 0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, 0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0xe0,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xde,0x00,0x00,0x00, 0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdf,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xb7,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, 0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xa8,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xc2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xfe,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0x0e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x17,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x18,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x1d,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x20,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0xb8,0x02,0x00,0x00,0x25,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0xb7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xbb,0x02,0x00,0x00,0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0x08,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x09,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x0b,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x06,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00,0x10,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x12,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x14,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x1b,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x1f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x21,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0xb7,0x02,0x00,0x00,0x26,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, 0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x29,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x70,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2b,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2f,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0xab,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00, -0x37,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x43,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x12,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x30,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0xb8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, +0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00, 0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x73,0x00,0x04,0x00, 0xc4,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, 0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x12,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00, 0x59,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x03,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, 0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x57,0x01,0x00,0x00, 0x3e,0x00,0x03,0x00,0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x12,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x50,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x61,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x06,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00,0x61,0x01,0x00,0x00, 0x60,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, 0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, 0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x63,0x01,0x00,0x00, 0x62,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x65,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x12,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x65,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00, -0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, -0x6e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00, -0xbc,0x02,0x00,0x00,0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0x75,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x2b,0x01,0x00,0x00,0x20,0x02,0x00,0x00,0x7c,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7f,0x01,0x00,0x00, -0xc2,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7b,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x7f,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x81,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x81,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x83,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x87,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x82,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0xaa,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8a,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x95,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00, -0x95,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x65,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00,0x67,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x6a,0x01,0x00,0x00,0x69,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, +0x6d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0xbb,0x02,0x00,0x00,0x71,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x76,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, +0x74,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x1f,0x02,0x00,0x00,0x7b,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0xc1,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x7a,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7e,0x01,0x00,0x00,0x79,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x80,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x79,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x82,0x01,0x00,0x00, +0x83,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x86,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x82,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x81,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0x89,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8e,0x01,0x00,0x00, +0x89,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x89,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x56,0x00,0x00,0x00, 0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x65,0x00,0x00,0x00, +0x9a,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x65,0x00,0x00,0x00, 0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9f,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0x9f,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa5,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0xfb,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, -0x93,0x01,0x00,0x00,0x97,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa8,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x89,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x84,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x84,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, -0xc6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x81,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xae,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x83,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xb4,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb0,0x01,0x00,0x00, -0xb1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb4,0x01,0x00,0x00,0xaf,0x01,0x00,0x00,0xb0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xaf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xaf,0x01,0x00,0x00,0xd8,0x01,0x00,0x00, -0xb7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xbc,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xb8,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbc,0x01,0x00,0x00, -0xb7,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc2,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc4,0x01,0x00,0x00, -0xc2,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0xd7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa7,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xd7,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x88,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x8a,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x83,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x83,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xc5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x82,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaf,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbb,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, 0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc9,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xc8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x69,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0xc7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0x69,0x00,0x00,0x00, 0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, -0xcd,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd3,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, -0xc0,0x01,0x00,0x00,0xc4,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xd6,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd5,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xda,0x01,0x00,0x00, -0xc7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xae,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xb0,0x01,0x00,0x00, -0x1e,0x02,0x00,0x00,0xdf,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xe2,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xde,0x01,0x00,0x00, -0xdf,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe2,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0x1c,0x02,0x00,0x00, -0xe7,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xea,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xe6,0x01,0x00,0x00,0xe7,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xea,0x01,0x00,0x00, -0xe5,0x01,0x00,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xe5,0x01,0x00,0x00,0x1a,0x02,0x00,0x00,0xef,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xce,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xee,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf2,0x01,0x00,0x00,0xed,0x01,0x00,0x00, -0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd0,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0x18,0x02,0x00,0x00,0xf5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf6,0x01,0x00,0x00, -0xf5,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xfa,0x01,0x00,0x00,0xf5,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0xcc,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xd5,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xd4,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xc6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xaf,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0x1d,0x02,0x00,0x00,0xde,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdd,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe1,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x1b,0x02,0x00,0x00, +0xe6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0xcb,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe5,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe9,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xe4,0x01,0x00,0x00,0x19,0x02,0x00,0x00,0xee,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf1,0x01,0x00,0x00, +0xcd,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf1,0x01,0x00,0x00,0xec,0x01,0x00,0x00, +0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcf,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0x17,0x02,0x00,0x00,0xf4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xcf,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf9,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, 0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfe,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0xce,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0xfe,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0xcd,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, 0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0x03,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0xd0,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0x0a,0x02,0x00,0x00,0x93,0x01,0x00,0x00,0x09,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, -0x0a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0x10,0x02,0x00,0x00,0xc0,0x01,0x00,0x00,0xfe,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x11,0x02,0x00,0x00, -0x10,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0x13,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x05,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x14,0x02,0x00,0x00, -0x13,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, -0x15,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x0b,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x14,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x13,0x02,0x00,0x00,0x15,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x02,0x00,0x00, -0xd0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf6,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xef,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xef,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xee,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe7,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1c,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, -0xc8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x20,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x02,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0x01,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0xcf,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x01,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x92,0x01,0x00,0x00,0x08,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x0f,0x02,0x00,0x00,0xbf,0x01,0x00,0x00,0xfd,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x10,0x02,0x00,0x00, +0x0f,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x13,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x14,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x13,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x12,0x02,0x00,0x00,0x14,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0xcb,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0xc7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, 0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0xa8,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, 0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x02,0x00,0x00, 0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x97,0x00,0x00,0x00, -0x27,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0xa8,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x14,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, -0x33,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x33,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, -0x39,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x34,0x02,0x00,0x00, -0x3c,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa9,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xd6,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0x42,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x45,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x41,0x02,0x00,0x00,0x42,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x45,0x02,0x00,0x00,0x40,0x02,0x00,0x00, -0x41,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x47,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xaa,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x40,0x02,0x00,0x00, -0xa4,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x49,0x02,0x00,0x00, -0x4a,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x4d,0x02,0x00,0x00,0x48,0x02,0x00,0x00,0x49,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x48,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x97,0x00,0x00,0x00, +0x26,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2c,0x02,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x39,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x39,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00,0x33,0x02,0x00,0x00, +0x3b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x41,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x40,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x44,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x46,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x46,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa9,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x3f,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0x49,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x48,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4c,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0x48,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x50,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, 0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, 0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x54,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xc8,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5c,0x02,0x00,0x00,0x69,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x58,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xc7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x69,0x00,0x00,0x00, 0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0x5c,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x48,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0xac,0x02,0x00,0x00, -0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x61,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x65,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0x61,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x60,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x6a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x69,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00, -0x68,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x68,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x70,0x02,0x00,0x00,0x55,0x02,0x00,0x00,0xae,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x73,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x75,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x73,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0x75,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x78,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x79,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x7b,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x75,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x75,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, -0x7c,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x68,0x02,0x00,0x00, -0x7b,0x02,0x00,0x00,0x74,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, -0x7e,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0xac,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x88,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, -0x88,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x89,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00,0x8b,0x02,0x00,0x00, -0x55,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xae,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0xa9,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0x91,0x02,0x00,0x00, -0xac,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x94,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0xaa,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x97,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9a,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0xae,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, -0xca,0x00,0x00,0x00,0x9a,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, -0x41,0x00,0x06,0x00,0x9d,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0x35,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0x9e,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0xae,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x67,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0xac,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x4a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x4a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa4,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x49,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x42,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x5c,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x5b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xab,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x47,0x02,0x00,0x00, +0xa1,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x64,0x02,0x00,0x00,0xab,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x60,0x02,0x00,0x00, +0x61,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x64,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x60,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, +0x69,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x6c,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x68,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6c,0x02,0x00,0x00, +0x67,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x67,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6f,0x02,0x00,0x00,0x54,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x72,0x02,0x00,0x00, +0x6f,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x74,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x74,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x5c,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x78,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x79,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x74,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x74,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x67,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7d,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0x7d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x5c,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x86,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, +0x87,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x89,0x02,0x00,0x00,0x85,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x3c,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x8a,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0x90,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x9b,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x9c,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0x35,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9d,0x02,0x00,0x00,0x9b,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x69,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x66,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x61,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, +0xab,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x49,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x49,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x46,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x48,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x41,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, 0x38,0x00,0x01,0x00, }; const uint64_t matmul_f16_aligned_fp32_len = 10288; @@ -14657,7 +14689,7 @@ const uint64_t matmul_f16_f32_len = 10332; unsigned char matmul_f16_f32_aligned_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xfd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x12,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, 0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, 0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, @@ -14665,9 +14697,9 @@ unsigned char matmul_f16_f32_aligned_data[] = { 0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, 0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, 0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x07,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x5e,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x73,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, @@ -14710,37 +14742,40 @@ unsigned char matmul_f16_f32_aligned_data[] = { 0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x05,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x05,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x03,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3c,0x01,0x00,0x00, 0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4e,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, +0x68,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x69,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x69,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x4f,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x69,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x47,0x00,0x03,0x00,0x69,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6b,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6b,0x01,0x00,0x00, 0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x5e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa4,0x02,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xa5,0x02,0x00,0x00, +0x73,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xba,0x02,0x00,0x00, 0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xa5,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xa5,0x02,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa7,0x02,0x00,0x00, +0xba,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xba,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x02,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa7,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0xbc,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, 0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, 0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -14791,16 +14826,16 @@ unsigned char matmul_f16_f32_aligned_data[] = { 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, 0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, 0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x06,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, @@ -14843,733 +14878,765 @@ unsigned char matmul_f16_f32_aligned_data[] = { 0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00, 0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, 0x15,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xf7,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0x16,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00,0x04,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x06,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x12,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x15,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x1b,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x3e,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, 0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0xf7,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x48,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x48,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x4c,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x4d,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, -0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x53,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xb3,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xb4,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xb3,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xca,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xe1,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0xe0,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xe2,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xe1,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf3,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x56,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xa4,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0xa6,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0xa6,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x61,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x62,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x62,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x66,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x67,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x69,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x6a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6d,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, 0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xb4,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xc7,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xdf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf6,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xb9,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xba,0x02,0x00,0x00, +0xb9,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xbb,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xbb,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, 0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe4,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0xe0,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdf,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xcc,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xc2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xdc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xff,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0xe0,0x02,0x00,0x00, -0x0e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, -0x16,0x01,0x00,0x00,0xfc,0x00,0x00,0x00,0x03,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x16,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x17,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0xdc,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, -0x1a,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, -0x20,0x01,0x00,0x00,0xfc,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe0,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0xdc,0x02,0x00,0x00, -0x25,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf9,0x02,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf5,0x02,0x00,0x00, +0x9f,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe1,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x5c,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, +0x8f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xf1,0x02,0x00,0x00,0x38,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xf1,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf5,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x16,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x13,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1e,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x23,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x25,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x2e,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x2f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x33,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0xf1,0x02,0x00,0x00, +0x3f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, 0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x2f,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x2b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2f,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xdd,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0xe4,0x02,0x00,0x00,0x38,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x39,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x34,0x01,0x00,0x00, -0x40,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, -0x41,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x53,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x43,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xa7,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x45,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x49,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x44,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xf2,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x4e,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0x52,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6d,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x35,0x00,0x00,0x00, 0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x53,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x5b,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x59,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x08,0x00,0x53,0x01,0x00,0x00,0x62,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x76,0x01,0x00,0x00, +0x75,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x77,0x01,0x00,0x00,0x76,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x78,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x78,0x01,0x00,0x00, +0x77,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00, 0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x62,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, -0x65,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x53,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x6d,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x6f,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x53,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x73,0x01,0x00,0x00, -0x72,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x70,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x75,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x78,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x77,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x53,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x7b,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0x7b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, -0x7d,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x78,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x53,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x85,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x87,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x53,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x67,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x7e,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x7f,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6d,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x86,0x01,0x00,0x00,0x85,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x8b,0x01,0x00,0x00, -0x8a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, +0x8a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, 0x8c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0x0a,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x63,0x01,0x00,0x00, 0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x8d,0x01,0x00,0x00, 0x8c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0xdd,0x02,0x00,0x00,0x90,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x93,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xe0,0x02,0x00,0x00, -0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0xe4,0x02,0x00,0x00,0x97,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x45,0x02,0x00,0x00,0x9e,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0xe6,0x02,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x9d,0x01,0x00,0x00, -0x9e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa1,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0xa6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xa9,0x01,0x00,0x00,0xea,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xa5,0x01,0x00,0x00,0xa6,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa9,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xab,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xfc,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xcd,0x01,0x00,0x00,0xac,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xb1,0x01,0x00,0x00, -0xfc,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xad,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb1,0x01,0x00,0x00,0xac,0x01,0x00,0x00, -0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xac,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, -0xea,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, -0xfc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x01,0x00,0x00, -0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, -0xbd,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc0,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, -0xbe,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, -0xfc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc5,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, -0xc5,0x01,0x00,0x00,0xe6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0xc9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xca,0x01,0x00,0x00,0xcb,0x01,0x00,0x00,0xb5,0x01,0x00,0x00, -0xb9,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xcb,0x01,0x00,0x00, -0xc9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xad,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0xea,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xeb,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0xfd,0x01,0x00,0x00, -0xd4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xd7,0x01,0x00,0x00,0xeb,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd3,0x01,0x00,0x00,0xd4,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd7,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0xd3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xf9,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0xda,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdf,0x01,0x00,0x00, -0xf9,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xdf,0x01,0x00,0x00,0xda,0x01,0x00,0x00, -0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xda,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe5,0x01,0x00,0x00, -0xeb,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, -0xf9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe9,0x01,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x01,0x00,0x00, -0xeb,0x02,0x00,0x00,0xeb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xef,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, -0xed,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00,0xf0,0x01,0x00,0x00, -0xf9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf4,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, -0xf4,0x01,0x00,0x00,0xe6,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x15,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0xf6,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0xf8,0x01,0x00,0x00,0xf7,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xca,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xf9,0x01,0x00,0x00, -0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfb,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xeb,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xff,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xec,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0x43,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x01,0x02,0x00,0x00,0x02,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x05,0x02,0x00,0x00, -0x00,0x02,0x00,0x00,0x01,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x00,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xf0,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x00,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0xf0,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x09,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x0d,0x02,0x00,0x00,0x08,0x02,0x00,0x00, -0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x08,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xf2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0xf2,0x02,0x00,0x00, -0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x11,0x02,0x00,0x00, -0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x15,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x11,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x10,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x17,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf4,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x1d,0x02,0x00,0x00,0xf4,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x19,0x02,0x00,0x00,0x18,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1d,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1f,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, -0x1f,0x02,0x00,0x00,0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00,0x21,0x02,0x00,0x00, -0x22,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0xf0,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x26,0x02,0x00,0x00, -0x23,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x26,0x02,0x00,0x00, -0xf4,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2c,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xca,0x01,0x00,0x00,0x2d,0x02,0x00,0x00, -0xb5,0x01,0x00,0x00,0x2c,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, -0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xca,0x01,0x00,0x00, -0x34,0x02,0x00,0x00,0xe3,0x01,0x00,0x00,0x21,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x35,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x36,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0xca,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x39,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xc4,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x36,0x02,0x00,0x00, -0x39,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x38,0x02,0x00,0x00, -0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x02,0x00,0x00,0xf4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x8f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x94,0x01,0x00,0x00,0x93,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x2e,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x6d,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x9a,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x96,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9b,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x6d,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa2,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0xa5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x43,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x45,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xf5,0x02,0x00,0x00, +0xa9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0xac,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfb,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x5a,0x02,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xfb,0x02,0x00,0x00, +0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb2,0x01,0x00,0x00, +0xb3,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb6,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xff,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0xbb,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0xff,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0xb9,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc6,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0xc2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xff,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0xff,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0xfb,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe0,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x11,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xff,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xba,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x00,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x12,0x02,0x00,0x00, +0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, +0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xef,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, +0x0e,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf4,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0x00,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, +0x0e,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x01,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x0e,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x08,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0xfb,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0x63,0x01,0x00,0x00, +0x0b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xdf,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xf8,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x0e,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x00,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x01,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0x58,0x02,0x00,0x00, +0x17,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x01,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x16,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1a,0x02,0x00,0x00, +0x15,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x15,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x1e,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x22,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, +0x1e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x07,0x03,0x00,0x00,0x3f,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x26,0x02,0x00,0x00, +0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2a,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x26,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x25,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x09,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x09,0x03,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x02,0x00,0x00,0x01,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x34,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x05,0x03,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x09,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x09,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00,0x42,0x02,0x00,0x00, +0xca,0x01,0x00,0x00,0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00, +0x49,0x02,0x00,0x00,0xf8,0x01,0x00,0x00,0x36,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0x3d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc4,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x4e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x4d,0x02,0x00,0x00, +0x4f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0x09,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x27,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x26,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x02,0x00,0x00, 0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x19,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x02,0x00,0x00,0xf2,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x0f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x11,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x0a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x02,0x00,0x00, -0xf0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x09,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x02,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x02,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x02,0x00,0x00,0xec,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x01,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0xe6,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x93,0x01,0x00,0x00, +0x17,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x58,0x02,0x00,0x00,0x01,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x16,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xfb,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, 0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, 0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, +0x5c,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, 0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, 0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4c,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4d,0x02,0x00,0x00, -0x97,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x5a,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x53,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x57,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0x56,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x57,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0x48,0x00,0x00,0x00, -0x58,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x5f,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x60,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x61,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x60,0x02,0x00,0x00, +0x61,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00, -0x59,0x02,0x00,0x00,0x61,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xca,0x02,0x00,0x00, -0x67,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x6a,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6a,0x02,0x00,0x00, -0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x65,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x65,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x72,0x02,0x00,0x00, -0xce,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x6e,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x6e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6d,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x76,0x02,0x00,0x00, -0xce,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x4d,0x02,0x00,0x00, -0x76,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x79,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, -0x77,0x02,0x00,0x00,0x79,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, -0xeb,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7f,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x7e,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x81,0x02,0x00,0x00, +0x97,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x5a,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x14,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x48,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x6e,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x7f,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0xdd,0x02,0x00,0x00,0x84,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0xe3,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x87,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0xe3,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x62,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, 0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x81,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0xc6,0x02,0x00,0x00,0x87,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, -0xd0,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x86,0x02,0x00,0x00,0x87,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x8a,0x02,0x00,0x00,0x85,0x02,0x00,0x00, -0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x85,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0xc4,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0xd2,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x8e,0x02,0x00,0x00, -0x8f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x92,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, -0xd2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x98,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x9a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x98,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x99,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0x14,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x9f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0x9d,0x02,0x00,0x00, -0x9f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xc2,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x98,0x02,0x00,0x00, -0x8d,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x99,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0xa3,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa1,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0xa3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa2,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x02,0x00,0x00, -0x82,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0x14,0x00,0x00,0x00, -0xac,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0xab,0x02,0x00,0x00, -0xae,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb0,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0xaf,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x99,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x9f,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa3,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa7,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xaf,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x02,0x00,0x00, 0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0xb0,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, -0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb6,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, -0xb6,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0xb8,0x02,0x00,0x00, -0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00, -0xba,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0xbd,0x02,0x00,0x00, -0xd2,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xc0,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, -0xc0,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x53,0x01,0x00,0x00, -0xc2,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0x35,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xc2,0x02,0x00,0x00, -0xc1,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xa3,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0xd2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x8c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8e,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x87,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x87,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc6,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x86,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0xce,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00, -0xcd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xaf,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xaf,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc2,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0xb8,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb6,0x02,0x00,0x00,0xb7,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0xc5,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0xcb,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0xce,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xd2,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0xd4,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xd5,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x6d,0x01,0x00,0x00, +0xd7,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x35,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xd7,0x02,0x00,0x00, +0xd6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa4,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x99,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0xe2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, 0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t matmul_f16_f32_aligned_len = 10940; +const uint64_t matmul_f16_f32_aligned_len = 11360; unsigned char matmul_f16_f32_aligned_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -15581,8 +15648,8 @@ unsigned char matmul_f16_f32_aligned_fp32_data[] = { 0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x14,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x36,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0xfd,0x00,0x00,0x00,0x04,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x36,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, @@ -15625,25 +15692,25 @@ unsigned char matmul_f16_f32_aligned_fp32_data[] = { 0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x02,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x21,0x01,0x00,0x00, +0x02,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x22,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4d,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4e,0x01,0x00,0x00, +0x23,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4e,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x4e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4e,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4f,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x51,0x01,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x50,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, 0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7c,0x02,0x00,0x00, 0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, @@ -15703,16 +15770,16 @@ unsigned char matmul_f16_f32_aligned_fp32_data[] = { 0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, 0x68,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00, 0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, 0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, 0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, 0x87,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, @@ -15756,677 +15823,678 @@ unsigned char matmul_f16_f32_aligned_fp32_data[] = { 0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, 0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, 0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0x03,0x01,0x00,0x00, -0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x12,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x21,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x51,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x1c,0x00,0x04,0x00,0x47,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x48,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x48,0x01,0x00,0x00,0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00, -0x17,0x00,0x04,0x00,0x4c,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x4d,0x01,0x00,0x00, -0x4c,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x4e,0x01,0x00,0x00, -0x4d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x4f,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x4e,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x4f,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x52,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x63,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x69,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6a,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, -0x8d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xff,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0xff,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x00,0x00, 0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xbb,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xba,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xbc,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xbb,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xcd,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xfc,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x2e,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x7c,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x7d,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x7e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x7d,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xbc,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xa3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbc,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, -0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0xe0,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdf,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xa4,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xc2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xfe,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, -0x0e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x17,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x18,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0xb4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x1d,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x20,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x20,0x01,0x00,0x00, -0xcc,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x27,0x01,0x00,0x00, -0xb4,0x02,0x00,0x00,0x25,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x29,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x2f,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, -0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2b,0x01,0x00,0x00, -0x2a,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2f,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, -0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0xab,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00, -0x37,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, -0x38,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, -0x34,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x43,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x52,0x01,0x00,0x00,0x53,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x44,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00, -0x54,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x52,0x01,0x00,0x00,0x59,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x57,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x5b,0x01,0x00,0x00, -0x5a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x41,0x00,0x07,0x00,0x52,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x5d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x01,0x00,0x00, -0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x63,0x01,0x00,0x00, -0x41,0x00,0x07,0x00,0x52,0x01,0x00,0x00,0x66,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x68,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0x64,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x68,0x01,0x00,0x00, -0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6d,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x6b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, -0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0x72,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x76,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xbe,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x2b,0x01,0x00,0x00, -0x1d,0x02,0x00,0x00,0x79,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x78,0x01,0x00,0x00, -0x79,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x7c,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x78,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0x81,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x84,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x80,0x01,0x00,0x00,0x81,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x84,0x01,0x00,0x00, -0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x7f,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, -0xd4,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x8c,0x01,0x00,0x00,0x87,0x01,0x00,0x00, -0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, -0xc2,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, -0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, -0xc2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0x99,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xfb,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x90,0x01,0x00,0x00, -0x94,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa5,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa7,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xab,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xae,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xb1,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xac,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0xd1,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, -0xc3,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, -0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc3,0x01,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, -0xc3,0x02,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc9,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xca,0x01,0x00,0x00, -0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x49,0x01,0x00,0x00, -0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xd3,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd5,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xae,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x1b,0x02,0x00,0x00, -0xdc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdf,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xda,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xda,0x01,0x00,0x00,0x19,0x02,0x00,0x00,0xe4,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, -0xc8,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, -0x17,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xca,0x02,0x00,0x00, -0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xeb,0x01,0x00,0x00, -0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x15,0x02,0x00,0x00, -0xf2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xf7,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, -0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, -0xf9,0x01,0x00,0x00,0xca,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, -0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x00,0x02,0x00,0x00, -0xcc,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x06,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0xcc,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x07,0x02,0x00,0x00, -0x90,0x01,0x00,0x00,0x06,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, -0xbd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0xca,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x12,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x08,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, -0xca,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x19,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, -0xbe,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x6e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x56,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x25,0x02,0x00,0x00,0x97,0x00,0x00,0x00,0x24,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, -0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0xa8,0x00,0x00,0x00, -0x2a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x2f,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x31,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x30,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00, -0x48,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x48,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x49,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x4d,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x53,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x8e,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xbb,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xbc,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x7c,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7d,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbc,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x35,0x02,0x00,0x00, -0x38,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3a,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x39,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3e,0x02,0x00,0x00, -0x3f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x42,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x44,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x47,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x4a,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x46,0x02,0x00,0x00,0x47,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4a,0x02,0x00,0x00, -0x45,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x45,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, -0x25,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x65,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x9f,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x8f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x38,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00,0x08,0x01,0x00,0x00, +0x07,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x0b,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x0b,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0x04,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x06,0x01,0x00,0x00, +0x16,0x01,0x00,0x00,0x04,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x06,0x01,0x00,0x00,0x1e,0x01,0x00,0x00,0x04,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xff,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x1e,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0a,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0xfd,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x21,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0xb4,0x02,0x00,0x00,0x26,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0xb5,0x02,0x00,0x00,0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x2c,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x30,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, +0x2c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x7f,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x35,0x01,0x00,0x00, +0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0x7a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x7a,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x53,0x01,0x00,0x00, +0x54,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x56,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x53,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x53,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x62,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x53,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x51,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, +0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0xb8,0x02,0x00,0x00,0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x1d,0x02,0x00,0x00,0x79,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0xbe,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x78,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7c,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x80,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x84,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x86,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x8c,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x65,0x00,0x00,0x00, 0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x52,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x51,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0xa5,0x02,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x62,0x02,0x00,0x00, -0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x5d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x67,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, -0xaa,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6a,0x02,0x00,0x00,0x65,0x02,0x00,0x00, -0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, -0x52,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x72,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x70,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x75,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x76,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x76,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x78,0x02,0x00,0x00, -0x75,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00,0x79,0x02,0x00,0x00, -0x70,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x78,0x02,0x00,0x00, -0x71,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x7b,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x83,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x85,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x85,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0x83,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, -0x87,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8a,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x52,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, -0x8a,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0x9c,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0x90,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xd4,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x81,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb1,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0xc5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x69,0x00,0x00,0x00, 0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x90,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00, -0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, +0xca,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0xcd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0a,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x1b,0x02,0x00,0x00,0xdc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00, +0xdc,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdf,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xda,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x19,0x02,0x00,0x00, +0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x17,0x02,0x00,0x00,0xec,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xef,0x01,0x00,0x00, +0xca,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xeb,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xea,0x01,0x00,0x00, +0x15,0x02,0x00,0x00,0xf2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xcc,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, +0xfb,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, 0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x94,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0xca,0x00,0x00,0x00, -0x97,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x99,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x52,0x01,0x00,0x00,0x9a,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, -0x35,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x67,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9c,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0xa6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x46,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3e,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - +0x00,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0xcc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x90,0x01,0x00,0x00,0x06,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0xbd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x11,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x17,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00, +0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x97,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x37,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00, +0x35,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x31,0x02,0x00,0x00, +0x39,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x44,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x46,0x02,0x00,0x00, +0x47,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4a,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x46,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x45,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0xc5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0x2b,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x69,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x45,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00, +0x5f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x62,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0x67,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x6a,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6a,0x02,0x00,0x00, +0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x65,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x70,0x02,0x00,0x00, +0x6d,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x72,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x78,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x72,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0x79,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x65,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7b,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x85,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x86,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0x87,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x92,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x98,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x53,0x01,0x00,0x00,0x9a,0x02,0x00,0x00, +0x7f,0x02,0x00,0x00,0x35,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x67,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x46,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, }; -const uint64_t matmul_f16_f32_aligned_fp32_len = 10224; +const uint64_t matmul_f16_f32_aligned_fp32_len = 10240; unsigned char matmul_f16_f32_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, @@ -19016,7 +19084,7 @@ const uint64_t matmul_f32_len = 10324; unsigned char matmul_f32_aligned_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xfd,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x17,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, @@ -19024,8 +19092,8 @@ unsigned char matmul_f32_aligned_data[] = { 0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, 0x14,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0xfc,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x5e,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, +0xfe,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x78,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, 0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, 0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, @@ -19068,37 +19136,4514 @@ unsigned char matmul_f32_aligned_data[] = { 0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, 0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x04,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x05,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x05,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x07,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x07,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x01,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x43,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x23,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6e,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, 0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x50,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x50,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x50,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x50,0x01,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x01,0x00,0x00, +0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x6f,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x71,0x01,0x00,0x00, 0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x52,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x5e,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa4,0x02,0x00,0x00, +0x71,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x78,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbe,0x02,0x00,0x00, 0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0xa5,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0xa5,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0xbf,0x02,0x00,0x00,0x00,0x00,0x00,0x00, 0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0xa5,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa7,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xa7,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0xbf,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xc1,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xc1,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf9,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xfd,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x46,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x46,0x01,0x00,0x00, +0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x69,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x6a,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x6e,0x01,0x00,0x00, +0x01,0x01,0x00,0x00,0x1e,0x00,0x03,0x00,0x6f,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x70,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x70,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xcd,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xce,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xcd,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xde,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xe4,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfa,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfb,0x01,0x00,0x00, +0xf9,0x00,0x00,0x00,0xfa,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0xfc,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3c,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x70,0x02,0x00,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x78,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xbe,0x02,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xbf,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0x20,0x00,0x04,0x00, +0xc0,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc0,0x02,0x00,0x00,0xc1,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xd3,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xce,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xfc,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x36,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x82,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0x4a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x4e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xe5,0x02,0x00,0x00,0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0xe5,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfe,0x02,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfa,0x02,0x00,0x00,0x9f,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xb0,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x85,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0xd7,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0xe6,0x02,0x00,0x00,0x8f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xf6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xf6,0x02,0x00,0x00, +0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0xf6,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xfa,0x02,0x00,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x11,0x01,0x00,0x00,0x10,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x1a,0x01,0x00,0x00, +0x19,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x20,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x22,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x28,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x30,0x01,0x00,0x00, +0x2f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x31,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x32,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x2d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x32,0x01,0x00,0x00, +0x31,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x39,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x42,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00,0xf6,0x02,0x00,0x00, +0x47,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x4b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x4b,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf7,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x51,0x01,0x00,0x00,0xf7,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x51,0x01,0x00,0x00, +0x4c,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xf7,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5b,0x01,0x00,0x00,0xfe,0x02,0x00,0x00,0x5a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0x62,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x65,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x63,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x76,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x76,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7d,0x01,0x00,0x00, +0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x82,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x83,0x01,0x00,0x00, +0x82,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x84,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x86,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x24,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x90,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x91,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x8d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x92,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x71,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x99,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x71,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0xa0,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa0,0x01,0x00,0x00,0x9f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0xa2,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa7,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xf7,0x02,0x00,0x00,0xaa,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4d,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb0,0x01,0x00,0x00,0xfa,0x02,0x00,0x00, +0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb3,0x01,0x00,0x00,0xfe,0x02,0x00,0x00,0xb1,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x00,0x03,0x00,0x00,0x3f,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x5f,0x02,0x00,0x00,0xb8,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x00,0x03,0x00,0x00, +0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00, +0xb8,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xbd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xbd,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x04,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0xc0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0x04,0x03,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xbf,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x01,0x00,0x00, +0xbe,0x01,0x00,0x00,0xbf,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xbe,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc5,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x16,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0x16,0x03,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc7,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xcb,0x01,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc6,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x01,0x00,0x00, +0x04,0x03,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x16,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0x04,0x03,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdb,0x01,0x00,0x00, +0xd8,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdd,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0x16,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xde,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe1,0x01,0x00,0x00, +0xdf,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0xe1,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xe3,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe5,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0x16,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x04,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xbd,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x05,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0x17,0x02,0x00,0x00, +0xee,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xf1,0x01,0x00,0x00,0x05,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf1,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x13,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x15,0x02,0x00,0x00,0xf4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf9,0x01,0x00,0x00, +0x13,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf5,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf9,0x01,0x00,0x00,0xf4,0x01,0x00,0x00, +0xf5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0x05,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0x13,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x03,0x02,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x03,0x02,0x00,0x00, +0x06,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0c,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, +0x13,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00, +0x0e,0x02,0x00,0x00,0x00,0x03,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x11,0x02,0x00,0x00,0x6b,0x01,0x00,0x00, +0x10,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xe4,0x01,0x00,0x00,0x13,0x02,0x00,0x00,0xfd,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x13,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x13,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00,0x05,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x06,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0xed,0x01,0x00,0x00,0x5d,0x02,0x00,0x00, +0x1c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0x06,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x1b,0x02,0x00,0x00,0x1c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1f,0x02,0x00,0x00, +0x1a,0x02,0x00,0x00,0x1b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x21,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x21,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0a,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x5b,0x02,0x00,0x00,0x24,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x27,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x23,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x27,0x02,0x00,0x00,0x22,0x02,0x00,0x00, +0x23,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x22,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x29,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x29,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x0c,0x03,0x00,0x00,0x3f,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x59,0x02,0x00,0x00,0x2c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2b,0x02,0x00,0x00, +0x2c,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2f,0x02,0x00,0x00,0x2a,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x31,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x31,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x0e,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x57,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x37,0x02,0x00,0x00,0x0e,0x03,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x33,0x02,0x00,0x00,0x32,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x37,0x02,0x00,0x00, +0x32,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x32,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x02,0x00,0x00,0x06,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x39,0x02,0x00,0x00,0x0c,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x3c,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x02,0x00,0x00,0x0a,0x03,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x02,0x00,0x00, +0x3d,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0x40,0x02,0x00,0x00, +0x0e,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x46,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x0e,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xe4,0x01,0x00,0x00,0x47,0x02,0x00,0x00, +0xcf,0x01,0x00,0x00,0x46,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x47,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x49,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xe4,0x01,0x00,0x00, +0x4e,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0x3b,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, +0x4e,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0x42,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x53,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc4,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x52,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x02,0x00,0x00,0x0e,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x31,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x33,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x0c,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x29,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x24,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0x0a,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x21,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x23,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x02,0x00,0x00,0x06,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x1b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x00,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x61,0x02,0x00,0x00,0xe6,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x97,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6c,0x02,0x00,0x00,0x5a,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x14,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x48,0x00,0x00,0x00, +0x72,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x79,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7b,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, +0x73,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe7,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xe4,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x84,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x80,0x02,0x00,0x00,0x81,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x84,0x02,0x00,0x00, +0x7f,0x02,0x00,0x00,0x80,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x7f,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0x89,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x88,0x02,0x00,0x00,0x89,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8c,0x02,0x00,0x00,0x87,0x02,0x00,0x00, +0x88,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00, +0xe8,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00,0x67,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, +0x91,0x02,0x00,0x00,0x93,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, +0x05,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0x6d,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x9e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x9e,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xea,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0xe0,0x02,0x00,0x00,0xa1,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0xea,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa0,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xa4,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa6,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xec,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0xde,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0xec,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa8,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xac,0x02,0x00,0x00,0xa7,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa7,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x02,0x00,0x00,0xaf,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xb4,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb2,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, +0x9c,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x02,0x00,0x00,0xb8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0xb7,0x02,0x00,0x00, +0xb9,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb4,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb4,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc2,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0xa7,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xb3,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0xbd,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xbb,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, +0xbd,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xbc,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00, +0x9c,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xc7,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc8,0x02,0x00,0x00,0xc7,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0xc5,0x02,0x00,0x00, +0xc8,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0xc9,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, +0xca,0x02,0x00,0x00,0x94,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0xe7,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xd0,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xd2,0x02,0x00,0x00, +0xd3,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd6,0x02,0x00,0x00,0xe8,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x02,0x00,0x00, +0xd4,0x02,0x00,0x00,0xd6,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00,0xd7,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xda,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xdb,0x02,0x00,0x00, +0xda,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, +0xdc,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0x35,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xdc,0x02,0x00,0x00, +0xdb,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xbd,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbd,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa9,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xde,0x02,0x00,0x00, +0xec,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa6,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa8,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0xea,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa0,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x89,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x89,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00,0xe8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x81,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe4,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f32_aligned_len = 11432; + +unsigned char matmul_f32_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xce,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0xfd,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x2f,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, +0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x38,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x61,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x00,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x01,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x01,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x01,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x1d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x1e,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x48,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x49,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x49,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x49,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4b,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4b,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x2f,0x02,0x00,0x00, +0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x75,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x76,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x76,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x76,0x02,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x78,0x02,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x78,0x02,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, +0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xfb,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfc,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0xff,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x08,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x1e,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x43,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x44,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x44,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x48,0x01,0x00,0x00,0xff,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x62,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x87,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x86,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x88,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x87,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xb4,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xb3,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xb5,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xb4,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x27,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x75,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x76,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x77,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x77,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, +0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xb5,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x31,0x00,0x00,0x00, +0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x55,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, +0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x9c,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00, +0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x18,0x02,0x00,0x00, +0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xdb,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0xde,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0xad,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0xb1,0x02,0x00,0x00,0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xed,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x05,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x09,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x05,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x0f,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x05,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x15,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x11,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x17,0x01,0x00,0x00,0x41,0x00,0x07,0x00,0x05,0x01,0x00,0x00, +0x1a,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0xad,0x02,0x00,0x00, +0x21,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x25,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x25,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0xae,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x27,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2b,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x26,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xae,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x34,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0x30,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x4f,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x54,0x01,0x00,0x00, +0x53,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x55,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, +0x59,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x5b,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5b,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x60,0x01,0x00,0x00, +0x5f,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x61,0x01,0x00,0x00,0x60,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0xae,0x02,0x00,0x00,0x64,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x25,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0xb1,0x02,0x00,0x00,0x68,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0xb5,0x02,0x00,0x00,0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x6f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb7,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x16,0x02,0x00,0x00, +0x72,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x75,0x01,0x00,0x00,0xb7,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x71,0x01,0x00,0x00,0x72,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x75,0x01,0x00,0x00, +0x70,0x01,0x00,0x00,0x71,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x70,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x70,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x7a,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0xbb,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7d,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcd,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0xcd,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x81,0x01,0x00,0x00, +0x80,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x85,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x8b,0x01,0x00,0x00,0xcd,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, +0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x91,0x01,0x00,0x00,0xbb,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0xcd,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x99,0x01,0x00,0x00, +0xb7,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x9e,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9e,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00, +0xcd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa4,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x79,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xaa,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xa6,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xaa,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, +0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xac,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xae,0x01,0x00,0x00, +0xad,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb2,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xb8,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0xb8,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, +0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc0,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc2,0x01,0x00,0x00, +0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, +0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc5,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xc7,0x01,0x00,0x00, +0xb7,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0xb6,0x01,0x00,0x00,0xba,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcc,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xca,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xac,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa7,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa6,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd2,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbd,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xa6,0x01,0x00,0x00,0x14,0x02,0x00,0x00,0xd5,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xd8,0x01,0x00,0x00, +0xbd,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xd4,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xd8,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xda,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xda,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0x12,0x02,0x00,0x00,0xdd,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0xc1,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdc,0x01,0x00,0x00, +0xdd,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe0,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xdb,0x01,0x00,0x00,0x10,0x02,0x00,0x00, +0xe5,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe8,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe8,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xe3,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xeb,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf0,0x01,0x00,0x00, +0xc5,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xec,0x01,0x00,0x00,0xeb,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf0,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, +0xbd,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf6,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x01,0x00,0x00, +0xc1,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0xc5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x89,0x01,0x00,0x00, +0xff,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x01,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xb6,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x06,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x09,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x0a,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc4,0x00,0x00,0x00,0x0b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x0a,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x09,0x02,0x00,0x00, +0x0b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0e,0x02,0x00,0x00,0xc5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x02,0x00,0x00,0xc3,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0xc1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xda,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd5,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x14,0x02,0x00,0x00,0xbd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x72,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x72,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00,0xb7,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x71,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x67,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1e,0x02,0x00,0x00, +0x97,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00,0x5a,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x23,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x28,0x02,0x00,0x00, +0x14,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x28,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x29,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x48,0x00,0x00,0x00, +0x29,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00, +0x30,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x31,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x02,0x00,0x00, +0x2a,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x35,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x3b,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x37,0x02,0x00,0x00,0x38,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3b,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x36,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x36,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0x40,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x43,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3f,0x02,0x00,0x00,0x40,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x43,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00, +0x9f,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x48,0x02,0x00,0x00,0x1e,0x02,0x00,0x00, +0x47,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4a,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, +0x48,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0xbe,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x50,0x02,0x00,0x00,0x24,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, +0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0x50,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x55,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x55,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3e,0x02,0x00,0x00,0x97,0x02,0x00,0x00,0x58,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x5b,0x02,0x00,0x00, +0xa1,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x57,0x02,0x00,0x00,0x58,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x5b,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x56,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0x60,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x63,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5f,0x02,0x00,0x00, +0x60,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x63,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x69,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x6b,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x69,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6a,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x6f,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x70,0x02,0x00,0x00,0x6f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x71,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, +0x70,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc2,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0x5e,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x6a,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0x74,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x72,0x02,0x00,0x00,0x73,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, +0x53,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x7e,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0x7d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x7f,0x02,0x00,0x00,0x7e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0x7f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x81,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0x80,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x83,0x02,0x00,0x00, +0x81,0x02,0x00,0x00,0x4b,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x02,0x00,0x00,0x83,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x9e,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x87,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00,0x89,0x02,0x00,0x00, +0x8a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8d,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x90,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x92,0x02,0x00,0x00, +0x91,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x05,0x01,0x00,0x00, +0x93,0x02,0x00,0x00,0x78,0x02,0x00,0x00,0x35,0x00,0x00,0x00, +0x85,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x93,0x02,0x00,0x00, +0x92,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x74,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x60,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, +0xa3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x58,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x58,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x55,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x57,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x9f,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x38,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x38,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x35,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f32_aligned_fp32_len = 10124; + +unsigned char matmul_f32_f16_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xd9,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x06,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x3a,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x38,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb8,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x04,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x06,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x22,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4f,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x50,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x50,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x50,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x52,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x52,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3a,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x80,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x81,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x81,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x81,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x83,0x02,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x83,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x91,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xba,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc1,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xf6,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x15,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x23,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x45,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x45,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4f,0x01,0x00,0x00,0xf6,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x51,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x50,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x52,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5d,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0x8e,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x90,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xa6,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xbd,0x01,0x00,0x00,0xf6,0x00,0x00,0x00, +0xbc,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xbd,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfe,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x80,0x02,0x00,0x00, +0xc3,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x81,0x02,0x00,0x00, +0x80,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x82,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0x82,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x88,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x95,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x90,0x01,0x00,0x00,0x91,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x73,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x78,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x81,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x8a,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x9d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xaf,0x00,0x00,0x00,0xae,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xa7,0x02,0x00,0x00, +0xc0,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc2,0x00,0x00,0x00,0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, +0xa7,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00, +0xcb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0xaf,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00, +0x9e,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x23,0x02,0x00,0x00,0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xda,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0xdf,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x38,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0x96,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xe6,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xed,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xeb,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0xa8,0x02,0x00,0x00,0x6f,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xf0,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xdd,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf3,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x00,0x01,0x00,0x00, +0x6f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x99,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x6f,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x16,0x01,0x00,0x00, +0xfb,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x16,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x17,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x74,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x1b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x1c,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0xfb,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0xb8,0x02,0x00,0x00,0x25,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, +0xb9,0x02,0x00,0x00,0xa6,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x2f,0x01,0x00,0x00,0x2a,0x01,0x00,0x00, +0x2b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, +0xa7,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x33,0x01,0x00,0x00, +0xb9,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x36,0x01,0x00,0x00,0x14,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x37,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x3a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x38,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x39,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0xa8,0x02,0x00,0x00,0x79,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x3d,0x01,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3a,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x2a,0x01,0x00,0x00,0x3f,0x01,0x00,0x00,0x39,0x01,0x00,0x00, +0xf7,0x00,0x03,0x00,0x42,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x40,0x01,0x00,0x00,0x41,0x01,0x00,0x00, +0x61,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x41,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4a,0x01,0x00,0x00, +0x7e,0x00,0x00,0x00,0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x4a,0x01,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, +0x59,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x5d,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x5f,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x42,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x64,0x01,0x00,0x00,0x7e,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00, +0x64,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x69,0x01,0x00,0x00,0x1f,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x42,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x42,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xb9,0x02,0x00,0x00, +0x6c,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x70,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, +0x73,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x21,0x02,0x00,0x00,0x7a,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x7d,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x79,0x01,0x00,0x00,0x7a,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7d,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x78,0x01,0x00,0x00, +0xab,0x01,0x00,0x00,0x82,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x85,0x01,0x00,0x00,0xc6,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x81,0x01,0x00,0x00, +0x82,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x85,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd8,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x89,0x01,0x00,0x00,0x88,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8d,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x89,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x88,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x93,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00, +0x93,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x01,0x00,0x00,0xc6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x01,0x00,0x00, +0x97,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x65,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x01,0x00,0x00,0x9a,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x9d,0x01,0x00,0x00,0xd8,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa1,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0xa1,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, +0xfb,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x91,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa7,0x01,0x00,0x00,0xa5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xd8,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x87,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x89,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x82,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xc6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xad,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x81,0x01,0x00,0x00, +0xd9,0x01,0x00,0x00,0xb0,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xb3,0x01,0x00,0x00,0xc7,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xaf,0x01,0x00,0x00, +0xb0,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb3,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xaf,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd5,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb7,0x01,0x00,0x00,0xb6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbb,0x01,0x00,0x00, +0xb6,0x01,0x00,0x00,0xb7,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc1,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc8,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xc7,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00, +0xc5,0x01,0x00,0x00,0xc8,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00,0x69,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcc,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00, +0xcc,0x01,0x00,0x00,0xd5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00,0xce,0x01,0x00,0x00, +0xcf,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x15,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf6,0x00,0x00,0x00,0xd4,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc3,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd5,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xd5,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00, +0xc7,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xaf,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xaf,0x01,0x00,0x00, +0x1f,0x02,0x00,0x00,0xde,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xe1,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdd,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xe1,0x01,0x00,0x00,0xdc,0x01,0x00,0x00,0xdd,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xdc,0x01,0x00,0x00,0x1d,0x02,0x00,0x00, +0xe6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xe9,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe5,0x01,0x00,0x00,0xe6,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe9,0x01,0x00,0x00, +0xe4,0x01,0x00,0x00,0xe5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xe4,0x01,0x00,0x00,0x1b,0x02,0x00,0x00,0xee,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xf1,0x01,0x00,0x00, +0xce,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xed,0x01,0x00,0x00,0xee,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf1,0x01,0x00,0x00,0xec,0x01,0x00,0x00, +0xed,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xd0,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xec,0x01,0x00,0x00, +0x19,0x02,0x00,0x00,0xf4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xd0,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf5,0x01,0x00,0x00, +0xf4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf9,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xf5,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf4,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0xce,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0xfe,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x02,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0x01,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x04,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x01,0x02,0x00,0x00, +0xd0,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xa6,0x01,0x00,0x00, +0x09,0x02,0x00,0x00,0x91,0x01,0x00,0x00,0x08,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00,0x0a,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x0b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xa6,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xbf,0x01,0x00,0x00, +0xfd,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf6,0x00,0x00,0x00, +0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x14,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0x14,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x16,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0x12,0x02,0x00,0x00,0x15,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x14,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0xce,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdd,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7a,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x21,0x02,0x00,0x00, +0xc2,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x77,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00, +0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x56,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x29,0x02,0x00,0x00,0x96,0x00,0x00,0x00,0x28,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, +0x5a,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x33,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x32,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0x33,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x34,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00, +0x48,0x00,0x00,0x00,0x34,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x3b,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3c,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x39,0x02,0x00,0x00, +0x3c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3e,0x02,0x00,0x00,0x35,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x40,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa9,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xa6,0x02,0x00,0x00,0x43,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x46,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x42,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x46,0x02,0x00,0x00,0x41,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x48,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x48,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x41,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x4b,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x4a,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4e,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x49,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00, +0x29,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x02,0x00,0x00,0x65,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0x53,0x02,0x00,0x00,0x55,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0xa9,0x02,0x00,0x00,0xc7,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5b,0x02,0x00,0x00,0x2f,0x02,0x00,0x00, +0x5a,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5d,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x02,0x00,0x00, +0x5b,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x60,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xac,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x49,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, +0x63,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x66,0x02,0x00,0x00,0xac,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x66,0x02,0x00,0x00, +0x61,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x61,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x68,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x61,0x02,0x00,0x00,0xa0,0x02,0x00,0x00,0x6b,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0xae,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x6a,0x02,0x00,0x00,0x6b,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6e,0x02,0x00,0x00,0x69,0x02,0x00,0x00, +0x6a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x69,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x02,0x00,0x00, +0x56,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x74,0x02,0x00,0x00,0x71,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x76,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x74,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x75,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x79,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x7a,0x02,0x00,0x00, +0x14,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x7b,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x7c,0x02,0x00,0x00, +0x79,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x76,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x76,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x7d,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0x69,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x7f,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7d,0x02,0x00,0x00, +0x7e,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x89,0x02,0x00,0x00, +0x14,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x89,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0x87,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x02,0x00,0x00,0x8c,0x02,0x00,0x00,0x56,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00, +0x8e,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00,0xa9,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0x96,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x02,0x00,0x00, +0x99,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0x9b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x9d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x11,0x01,0x00,0x00,0x9e,0x02,0x00,0x00,0x83,0x02,0x00,0x00, +0x35,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9e,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7f,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x6b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6b,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x68,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x6a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x63,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xac,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x60,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x4b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x4b,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x48,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x4a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x43,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x43,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa6,0x02,0x00,0x00,0xa9,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x40,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x42,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, + +}; +const uint64_t matmul_f32_f16_len = 10332; + +unsigned char matmul_f32_f16_aligned_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x12,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x09,0x00,0x00,0x00, +0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00,0x0b,0x00,0x06,0x00, +0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64, +0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00, +0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00, +0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e, +0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x05,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0x73,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x10,0x00,0x06,0x00, +0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x38,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x3e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x54,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x61,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x6d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x03,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x03,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x43,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x44,0x01,0x00,0x00, +0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x70,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x71,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x05,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x71,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x71,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x71,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x71,0x01,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x73,0x01,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x73,0x01,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x73,0x02,0x00,0x00,0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xb9,0x02,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xba,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xba,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xba,0x02,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x02,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xbc,0x02,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00, +0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x59,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x92,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbd,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0xf9,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0xfb,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xfc,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xfd,0x00,0x00,0x00,0xfe,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x00,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x01,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x04,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x03,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x07,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0b,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x05,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x43,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00,0x44,0x01,0x00,0x00, +0x43,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x46,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x47,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x67,0x01,0x00,0x00, +0x1c,0x00,0x04,0x00,0x69,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x6a,0x01,0x00,0x00, +0x04,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, +0x6a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x17,0x00,0x04,0x00,0x6e,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00,0x6f,0x01,0x00,0x00, +0x6e,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x70,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x71,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x72,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x72,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x75,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa3,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa3,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xac,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xc8,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xc7,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xc8,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xd9,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xdf,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf5,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf6,0x01,0x00,0x00,0xf9,0x00,0x00,0x00, +0xf5,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xf6,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x73,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xb9,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xba,0x02,0x00,0x00, +0xb9,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0xbb,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0xba,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0xbb,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xc1,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xce,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0xf8,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xe0,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xe0,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf9,0x02,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0xae,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf5,0x02,0x00,0x00, +0x9f,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xab,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe1,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x5c,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xe1,0x02,0x00,0x00, +0x8f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf1,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xf1,0x02,0x00,0x00,0x38,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xf1,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf5,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x09,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x0c,0x01,0x00,0x00,0x0a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x11,0x01,0x00,0x00, +0x10,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x12,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x0e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x13,0x01,0x00,0x00, +0x12,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x19,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x1a,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x20,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x22,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x24,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x2a,0x01,0x00,0x00, +0x29,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00,0x2c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x2f,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x31,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0xfe,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x34,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x07,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x05,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0xfe,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x3a,0x01,0x00,0x00,0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x07,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x40,0x01,0x00,0x00, +0x3f,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x41,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x42,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0xf1,0x02,0x00,0x00,0x47,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x4b,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x4b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xdf,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0xf2,0x02,0x00,0x00,0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x4c,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x4c,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00, +0x7f,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x56,0x01,0x00,0x00, +0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00, +0xf9,0x02,0x00,0x00,0x5a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5b,0x01,0x00,0x00, +0x7a,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0x7a,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x63,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x75,0x01,0x00,0x00, +0x76,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0x76,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x78,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00, +0x75,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x7d,0x01,0x00,0x00,0x7c,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x7a,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7e,0x01,0x00,0x00, +0x7d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x80,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x41,0x00,0x08,0x00,0x75,0x01,0x00,0x00,0x82,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x84,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x84,0x01,0x00,0x00,0x83,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x86,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x75,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x89,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x8a,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x86,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x8a,0x01,0x00,0x00,0x89,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x75,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x8f,0x01,0x00,0x00,0x8e,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x8c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x90,0x01,0x00,0x00, +0x8f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, +0x41,0x00,0x08,0x00,0x75,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x73,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0x94,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x6b,0x01,0x00,0x00,0x92,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x96,0x01,0x00,0x00,0x95,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x75,0x01,0x00,0x00, +0x9a,0x01,0x00,0x00,0x73,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x9b,0x01,0x00,0x00, +0x9a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x0b,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0x6b,0x01,0x00,0x00,0x98,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9c,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x41,0x00,0x08,0x00, +0x75,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x73,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, +0x1c,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xa1,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0xa2,0x01,0x00,0x00,0x6b,0x01,0x00,0x00, +0x9e,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa2,0x01,0x00,0x00, +0xa1,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0xf2,0x02,0x00,0x00,0xa5,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x4b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x4d,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xa8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xab,0x01,0x00,0x00,0xf5,0x02,0x00,0x00, +0xa9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0xac,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xfb,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x4d,0x01,0x00,0x00, +0x5a,0x02,0x00,0x00,0xb3,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xfb,0x02,0x00,0x00, +0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb2,0x01,0x00,0x00, +0xb3,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb6,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb1,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xff,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0xbb,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xbe,0x01,0x00,0x00,0xff,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xba,0x01,0x00,0x00,0xbb,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xbe,0x01,0x00,0x00, +0xb9,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb9,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x11,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0xe2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xc2,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xc6,0x01,0x00,0x00,0xc1,0x01,0x00,0x00, +0xc2,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xc1,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xff,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, +0xff,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xd0,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd6,0x01,0x00,0x00, +0xd3,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd8,0x01,0x00,0x00,0xd6,0x01,0x00,0x00, +0x11,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0xfb,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, +0xdc,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0xde,0x01,0x00,0x00,0xdd,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xe0,0x01,0x00,0x00, +0xde,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x11,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xc0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xc2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xbb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xbb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xff,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xba,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x00,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x12,0x02,0x00,0x00, +0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, +0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x0e,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0x10,0x02,0x00,0x00,0xef,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, +0x0e,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf4,0x01,0x00,0x00,0xef,0x01,0x00,0x00, +0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfa,0x01,0x00,0x00, +0x00,0x03,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, +0x0e,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfe,0x01,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x01,0x02,0x00,0x00, +0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0xfe,0x01,0x00,0x00, +0x01,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x04,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x05,0x02,0x00,0x00, +0x02,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x07,0x02,0x00,0x00,0x05,0x02,0x00,0x00, +0x0e,0x03,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x09,0x02,0x00,0x00,0x07,0x02,0x00,0x00,0x08,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x02,0x00,0x00, +0x09,0x02,0x00,0x00,0xfb,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0b,0x01,0x00,0x00,0x0c,0x02,0x00,0x00,0x6b,0x01,0x00,0x00, +0x0b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0x0c,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xdf,0x01,0x00,0x00,0x0e,0x02,0x00,0x00,0xf8,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0x0e,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x12,0x02,0x00,0x00,0x00,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x14,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x01,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0xe8,0x01,0x00,0x00,0x58,0x02,0x00,0x00, +0x17,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x1a,0x02,0x00,0x00,0x01,0x03,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x16,0x02,0x00,0x00,0x17,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x1a,0x02,0x00,0x00, +0x15,0x02,0x00,0x00,0x16,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x15,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x1c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x1c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x05,0x03,0x00,0x00,0x3f,0x00,0x00,0x00, +0x15,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x1f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x22,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x1e,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x22,0x02,0x00,0x00,0x1d,0x02,0x00,0x00, +0x1e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x24,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x07,0x03,0x00,0x00,0x3f,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0x54,0x02,0x00,0x00,0x27,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x26,0x02,0x00,0x00, +0x27,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2a,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x26,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x25,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x2c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x09,0x03,0x00,0x00, +0x3f,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x32,0x02,0x00,0x00,0x09,0x03,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x2e,0x02,0x00,0x00,0x2d,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x32,0x02,0x00,0x00, +0x2d,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x34,0x02,0x00,0x00,0x01,0x03,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x34,0x02,0x00,0x00,0x07,0x03,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x36,0x02,0x00,0x00, +0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x05,0x03,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0x3b,0x02,0x00,0x00, +0x09,0x03,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x41,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x09,0x03,0x00,0x00, +0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00,0x42,0x02,0x00,0x00, +0xca,0x01,0x00,0x00,0x41,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0x42,0x02,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x44,0x02,0x00,0x00, +0x43,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xdf,0x01,0x00,0x00, +0x49,0x02,0x00,0x00,0xf8,0x01,0x00,0x00,0x36,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xf9,0x00,0x00,0x00,0x4a,0x02,0x00,0x00, +0x49,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x4b,0x02,0x00,0x00,0x4a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0x4d,0x02,0x00,0x00,0xca,0x00,0x00,0x00, +0x3d,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0x4d,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, +0xc4,0x00,0x00,0x00,0x4f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x44,0x02,0x00,0x00,0x4b,0x02,0x00,0x00, +0x4e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x4d,0x02,0x00,0x00, +0x4f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x52,0x02,0x00,0x00,0x09,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x2e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x27,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x27,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x54,0x02,0x00,0x00,0x07,0x03,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x24,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x26,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0x05,0x03,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x1c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x1e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x17,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x58,0x02,0x00,0x00,0x01,0x03,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x14,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x16,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00,0xfb,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xa8,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5c,0x02,0x00,0x00,0xe1,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x61,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x62,0x02,0x00,0x00, +0x97,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0x5a,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x68,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x67,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x6c,0x02,0x00,0x00, +0x14,0x00,0x00,0x00,0x6b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x6c,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6e,0x02,0x00,0x00, +0x0f,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00,0x48,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, +0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00, +0x74,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0x75,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x6e,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe2,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x7f,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x7b,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x7f,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x7a,0x02,0x00,0x00,0xdd,0x02,0x00,0x00,0x84,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0xe3,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x83,0x02,0x00,0x00,0x84,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x87,0x02,0x00,0x00,0x82,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x82,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x02,0x00,0x00, +0xe3,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x62,0x02,0x00,0x00, +0x8b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8e,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00, +0x8c,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x93,0x02,0x00,0x00,0xe2,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x94,0x02,0x00,0x00,0x68,0x02,0x00,0x00,0x93,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x02,0x00,0x00, +0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x96,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x99,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x99,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xe5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x82,0x02,0x00,0x00,0xdb,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, +0xe5,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x9b,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x9f,0x02,0x00,0x00,0x9a,0x02,0x00,0x00, +0x9b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xe7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x9a,0x02,0x00,0x00, +0xd9,0x02,0x00,0x00,0xa4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xa7,0x02,0x00,0x00,0xe7,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa3,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xa7,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xad,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x37,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xaf,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xad,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xaf,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xb3,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0xb3,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0xb2,0x02,0x00,0x00, +0xb4,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xaf,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xaf,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc2,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xad,0x02,0x00,0x00, +0xa2,0x02,0x00,0x00,0xb5,0x02,0x00,0x00,0xae,0x02,0x00,0x00, +0xf7,0x00,0x03,0x00,0xb8,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb6,0x02,0x00,0x00,0xb7,0x02,0x00,0x00, +0xb8,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xb7,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00, +0x97,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xc2,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0xc1,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0xc2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00,0xc0,0x02,0x00,0x00, +0xc3,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc5,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, +0xc5,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xcb,0x02,0x00,0x00,0xe2,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcd,0x02,0x00,0x00, +0xcb,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcf,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, +0xce,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x02,0x00,0x00,0xe3,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00, +0xcf,0x02,0x00,0x00,0xd1,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0xd2,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0xd5,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0xd4,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xd6,0x02,0x00,0x00, +0xd5,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x07,0x01,0x00,0x00, +0xd7,0x02,0x00,0x00,0xbc,0x02,0x00,0x00,0x35,0x00,0x00,0x00, +0xc9,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xd7,0x02,0x00,0x00, +0xd6,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0xb8,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb8,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa4,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa4,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd9,0x02,0x00,0x00, +0xe7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xa1,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x9c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xdb,0x02,0x00,0x00,0xe5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x99,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x9b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xdd,0x02,0x00,0x00,0xe3,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x83,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xdf,0x02,0x00,0x00, +0xe2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x79,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, +0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f32_f16_aligned_len = 11360; + +unsigned char matmul_f32_f16_aligned_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xd5,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xfd,0x00,0x00,0x00,0x03,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x36,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x38,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x61,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x63,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa7,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x00,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x01,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x01,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x01,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x03,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x1d,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x1e,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4a,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4b,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4b,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4b,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4d,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x4d,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7c,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x7d,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x7d,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x7d,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x7f,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7f,0x02,0x00,0x00,0x21,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x10,0x00,0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x13,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x35,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x68,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x62,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x74,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x87,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x50,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0xf9,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfb,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xfc,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xfc,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0xff,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x00,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x01,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x02,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x02,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x05,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x08,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x17,0x01,0x00,0x00, +0x03,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x43,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x44,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x44,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x48,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x17,0x00,0x04,0x00, +0x49,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x4a,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0x1e,0x00,0x03,0x00,0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x4c,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x4b,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x4c,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x4f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00, +0x51,0x00,0x00,0x00,0x1e,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x6e,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x6d,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0x8e,0x01,0x00,0x00, +0xc4,0x00,0x00,0x00,0x8d,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xbb,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, +0xba,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0xbc,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x2e,0x02,0x00,0x00,0x0d,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x36,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x7c,0x02,0x00,0x00, +0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x7d,0x02,0x00,0x00, +0x7c,0x02,0x00,0x00,0x20,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, +0x0c,0x00,0x00,0x00,0x7d,0x02,0x00,0x00,0x3b,0x00,0x04,0x00, +0x7e,0x02,0x00,0x00,0x7f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0x05,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x91,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xc9,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xbc,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x16,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x2a,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x2f,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x31,0x00,0x00,0x00,0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x35,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x37,0x00,0x00,0x00,0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x3b,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x41,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3c,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x3e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x4d,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00, +0x55,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00, +0x64,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x7a,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x8b,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x26,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00, +0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x99,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x95,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9f,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x0f,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xac,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00, +0x85,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xaf,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0xc1,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xb0,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00, +0x9f,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0x1f,0x02,0x00,0x00,0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdb,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x8f,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdb,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0xde,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x38,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xde,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xe8,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0xf4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x07,0x01,0x00,0x00, +0x06,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x09,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, +0x3e,0x00,0x03,0x00,0x09,0x01,0x00,0x00,0x07,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x0d,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x0f,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x0b,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x11,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x13,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x13,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x15,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x11,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0xf8,0x00,0x00,0x00,0x17,0x01,0x00,0x00,0x41,0x00,0x07,0x00, +0x05,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x03,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x17,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, +0x1a,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x1c,0x01,0x00,0x00,0xfd,0x00,0x00,0x00,0x18,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1c,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0xb4,0x02,0x00,0x00,0x21,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x25,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x25,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x2b,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, +0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x27,0x01,0x00,0x00, +0x26,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x2b,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x27,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x26,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, +0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x33,0x01,0x00,0x00,0x30,0x01,0x00,0x00,0xab,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x34,0x01,0x00,0x00, +0x33,0x01,0x00,0x00,0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0x34,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0x3c,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, +0x6e,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x40,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x3f,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x4f,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x48,0x01,0x00,0x00, +0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x53,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x53,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00,0x4f,0x01,0x00,0x00, +0x57,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x48,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x59,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00, +0x5a,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x55,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x5a,0x01,0x00,0x00,0x59,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, +0x4f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00, +0x35,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x48,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x5e,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x08,0x01,0x00,0x00,0x61,0x01,0x00,0x00,0x45,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x61,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x17,0x01,0x00,0x00, +0x41,0x00,0x07,0x00,0x4f,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x4d,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x37,0x01,0x00,0x00, +0x17,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x48,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x66,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x68,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, +0x6b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x25,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x27,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, +0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, +0xb8,0x02,0x00,0x00,0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0xbc,0x02,0x00,0x00, +0x72,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xbe,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x1d,0x02,0x00,0x00,0x79,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, +0xbe,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x78,0x01,0x00,0x00,0x79,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x7c,0x01,0x00,0x00,0x77,0x01,0x00,0x00, +0x78,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x77,0x01,0x00,0x00, +0xa9,0x01,0x00,0x00,0x81,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x84,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x80,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x84,0x01,0x00,0x00,0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x86,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0xa7,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x8c,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, +0x87,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x87,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00, +0x92,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0x56,0x00,0x00,0x00, +0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x98,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00, +0x96,0x01,0x00,0x00,0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x01,0x00,0x00,0x65,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9c,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0x9b,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9c,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00, +0x9f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0xa0,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0xa2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, +0x90,0x01,0x00,0x00,0x94,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xa5,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa7,0x01,0x00,0x00,0xd4,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x81,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x80,0x01,0x00,0x00, +0xd7,0x01,0x00,0x00,0xae,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xb1,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xb1,0x01,0x00,0x00,0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xac,0x01,0x00,0x00,0xd5,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xb9,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb9,0x01,0x00,0x00, +0xb4,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb4,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xbf,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00, +0xbf,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc6,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0xc5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00, +0xc3,0x01,0x00,0x00,0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0x69,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xca,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xc9,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00, +0xca,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00, +0xcd,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd0,0x01,0x00,0x00,0xce,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x08,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, +0xbd,0x01,0x00,0x00,0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0xd3,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd5,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xae,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xc4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xad,0x01,0x00,0x00, +0x1b,0x02,0x00,0x00,0xdc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, +0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00, +0xdc,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xdf,0x01,0x00,0x00,0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xda,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xda,0x01,0x00,0x00,0x19,0x02,0x00,0x00, +0xe4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0xe7,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00, +0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xca,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xe2,0x01,0x00,0x00,0x17,0x02,0x00,0x00,0xec,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xef,0x01,0x00,0x00, +0xca,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xeb,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00, +0xeb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xcc,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xea,0x01,0x00,0x00, +0x15,0x02,0x00,0x00,0xf2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0xf7,0x01,0x00,0x00,0xcc,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xf7,0x01,0x00,0x00,0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf2,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf9,0x01,0x00,0x00,0xc4,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xfb,0x01,0x00,0x00,0xf9,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, +0xfb,0x01,0x00,0x00,0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x01,0x00,0x00,0xc8,0x02,0x00,0x00, +0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x00,0x02,0x00,0x00,0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x00,0x02,0x00,0x00,0xcc,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x06,0x02,0x00,0x00,0xff,0x01,0x00,0x00, +0xcc,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x07,0x02,0x00,0x00,0x90,0x01,0x00,0x00,0x06,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x07,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x0d,0x02,0x00,0x00,0xbd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x0e,0x02,0x00,0x00, +0x0d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x10,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x02,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x11,0x02,0x00,0x00, +0x10,0x02,0x00,0x00,0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00, +0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x08,0x02,0x00,0x00,0x0e,0x02,0x00,0x00,0x11,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x17,0x02,0x00,0x00,0xca,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xeb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x19,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdc,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00, +0xc4,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x79,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x1d,0x02,0x00,0x00,0xbe,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0xa4,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00, +0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0x97,0x00,0x00,0x00, +0x24,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00, +0xa8,0x00,0x00,0x00,0x2a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x2f,0x02,0x00,0x00,0x14,0x00,0x00,0x00, +0x2e,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x31,0x02,0x00,0x00,0x0f,0x00,0x00,0x00, +0x30,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x35,0x02,0x00,0x00,0x48,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00,0x37,0x02,0x00,0x00, +0x36,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00, +0x35,0x02,0x00,0x00,0x38,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3a,0x02,0x00,0x00,0x31,0x02,0x00,0x00, +0x39,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0x3f,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x42,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x3e,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x42,0x02,0x00,0x00,0x3d,0x02,0x00,0x00, +0x3e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x44,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x02,0x00,0x00, +0xa0,0x02,0x00,0x00,0x47,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x4a,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, +0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x46,0x02,0x00,0x00, +0x47,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x4a,0x02,0x00,0x00,0x45,0x02,0x00,0x00,0x46,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x45,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4e,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, +0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4f,0x02,0x00,0x00,0x25,0x02,0x00,0x00,0x4e,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00, +0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, +0x51,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x56,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0xc5,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00, +0x2b,0x02,0x00,0x00,0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x59,0x02,0x00,0x00,0x69,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x02,0x00,0x00,0x57,0x02,0x00,0x00,0x59,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x45,0x02,0x00,0x00, +0x9e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc2,0x00,0x00,0x00,0x62,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00, +0x5f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x62,0x02,0x00,0x00,0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x5d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00, +0x67,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x6a,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x6a,0x02,0x00,0x00, +0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x65,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x02,0x00,0x00,0x52,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x70,0x02,0x00,0x00, +0x6d,0x02,0x00,0x00,0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, +0x72,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x70,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x76,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, +0x76,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, +0x78,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x77,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x72,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00, +0x79,0x02,0x00,0x00,0x70,0x02,0x00,0x00,0x65,0x02,0x00,0x00, +0x78,0x02,0x00,0x00,0x71,0x02,0x00,0x00,0xf7,0x00,0x03,0x00, +0x7b,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x79,0x02,0x00,0x00,0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x83,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x85,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x84,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00, +0x85,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x87,0x02,0x00,0x00,0x83,0x02,0x00,0x00,0x86,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00, +0x3a,0x02,0x00,0x00,0x87,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8a,0x02,0x00,0x00,0x88,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8c,0x02,0x00,0x00,0x8a,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x90,0x02,0x00,0x00,0x8e,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x92,0x02,0x00,0x00,0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00,0x92,0x02,0x00,0x00, +0x94,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x95,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x98,0x02,0x00,0x00, +0xca,0x00,0x00,0x00,0x97,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc4,0x00,0x00,0x00,0x99,0x02,0x00,0x00,0x98,0x02,0x00,0x00, +0x41,0x00,0x06,0x00,0x05,0x01,0x00,0x00,0x9a,0x02,0x00,0x00, +0x7f,0x02,0x00,0x00,0x35,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x67,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9c,0x02,0x00,0x00,0xaa,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x66,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, +0xa8,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x47,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x46,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3e,0x02,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t matmul_f32_f16_aligned_fp32_len = 10240; + +unsigned char matmul_f32_f16_fp32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0xd5,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, +0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, +0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, +0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0x50,0x01,0x00,0x00,0x36,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x24,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x38,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x3e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4d,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x61,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x63,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x6d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xa6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xb8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xbb,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x02,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x03,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x03,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x05,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x21,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x1e,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x1f,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x4d,0x01,0x00,0x00,0x06,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x4e,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x4e,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x4e,0x01,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x50,0x01,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x50,0x01,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x36,0x02,0x00,0x00,0x0b,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7c,0x02,0x00,0x00, +0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00, +0x7d,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x7d,0x02,0x00,0x00,0x00,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x7d,0x02,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x7f,0x02,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x7f,0x02,0x00,0x00,0x21,0x00,0x00,0x00, 0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, 0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, 0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, @@ -19153,156 +23698,146 @@ unsigned char matmul_f32_aligned_data[] = { 0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00,0x86,0x00,0x00,0x00, 0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, 0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xf7,0x00,0x00,0x00, -0x10,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0xf8,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xfb,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xfa,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xfb,0x00,0x00,0x00,0xfc,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x04,0x01,0x00,0x00, -0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x05,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x06,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x05,0x01,0x00,0x00,0x3b,0x00,0x04,0x00, -0x06,0x01,0x00,0x00,0x07,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x12,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x16,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x24,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x25,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x48,0x01,0x00,0x00, -0xf7,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x49,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x49,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x4d,0x01,0x00,0x00, -0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x18,0x00,0x04,0x00, -0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x4f,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x1e,0x00,0x03,0x00,0x50,0x01,0x00,0x00,0x4f,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0x51,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x50,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x51,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x77,0x01,0x00,0x00,0x05,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x7f,0x01,0x00,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x87,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8e,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x8f,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x8e,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x90,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x8f,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0x08,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x86,0x00,0x00,0x00, 0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x97,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb2,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xb3,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0xb2,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xb4,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xb3,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xca,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xe0,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xe1,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0xe0,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xe2,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xe1,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xeb,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf3,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x81,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x50,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xba,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbc,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x86,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x14,0x00,0x02,0x00, +0xc1,0x00,0x00,0x00,0x16,0x00,0x03,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc4,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc6,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00,0xc7,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xc8,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0xf8,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xf9,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xf9,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, +0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x02,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x03,0x01,0x00,0x00,0x02,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, +0x03,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x04,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x10,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x13,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x19,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x3a,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00, +0x09,0x00,0x00,0x00,0x1f,0x01,0x00,0x00,0x1e,0x01,0x00,0x00, +0x3a,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x20,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x20,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x22,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x21,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x41,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x1c,0x00,0x04,0x00, +0x42,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0x41,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0x43,0x01,0x00,0x00,0x04,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x3b,0x00,0x04,0x00,0x43,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x48,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x4c,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x4d,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, +0x4e,0x01,0x00,0x00,0x4d,0x01,0x00,0x00,0x20,0x00,0x04,0x00, +0x4f,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x3b,0x00,0x04,0x00,0x4f,0x01,0x00,0x00,0x50,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x5b,0x01,0x00,0x00, +0x0c,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x51,0x00,0x00,0x00, +0x1f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x86,0x00,0x00,0x00, +0x6a,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x08,0x01,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x72,0x01,0x00,0x00, +0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8d,0x01,0x00,0x00, +0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x8e,0x01,0x00,0x00,0xc3,0x00,0x00,0x00, +0x8d,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x8f,0x01,0x00,0x00, +0x07,0x00,0x00,0x00,0x8e,0x01,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x80,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0xba,0x01,0x00,0x00,0x84,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0xbb,0x01,0x00,0x00,0xc3,0x00,0x00,0x00,0xba,0x01,0x00,0x00, +0x20,0x00,0x04,0x00,0xbc,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0xbb,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xc5,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0xcd,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, 0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x22,0x02,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, +0xfc,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, 0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x56,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x5e,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xa4,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xa5,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0xa6,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0xa5,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0xa6,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xb9,0x02,0x00,0x00, +0x2e,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x0a,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x7c,0x02,0x00,0x00,0xc3,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x7d,0x02,0x00,0x00,0x7c,0x02,0x00,0x00, +0x20,0x00,0x04,0x00,0x7e,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, +0x7d,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x7e,0x02,0x00,0x00, +0x7f,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x15,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x05,0x00,0x00,0x00, +0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x91,0x02,0x00,0x00, 0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, 0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xb4,0x01,0x00,0x00,0xb5,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xe2,0x01,0x00,0x00,0xe3,0x01,0x00,0x00, +0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc8,0x00,0x00,0x00, +0xc9,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x8f,0x01,0x00,0x00,0x90,0x01,0x00,0x00,0x07,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xbc,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, 0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, 0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, @@ -19362,1425 +23897,530 @@ unsigned char matmul_f32_aligned_data[] = { 0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, 0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, 0x74,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x79,0x00,0x00,0x00, +0x4f,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, +0x7d,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x81,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x82,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x84,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x88,0x00,0x00,0x00,0x87,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, 0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xcb,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xcb,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xcb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe4,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x99,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe0,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x96,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xcc,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xdc,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x28,0x01,0x00,0x00,0xe0,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdf,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xcc,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xc2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xdc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00, -0x00,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, -0xff,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0f,0x01,0x00,0x00,0xe0,0x02,0x00,0x00, -0x0e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x12,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0x07,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x13,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x14,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x17,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x17,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x18,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x74,0x00,0x00,0x00, -0xdc,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1d,0x01,0x00,0x00,0x1b,0x01,0x00,0x00,0x1c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x1d,0x01,0x00,0x00,0x6f,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0xfc,0x00,0x00,0x00, -0x1f,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x21,0x01,0x00,0x00, -0x20,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe0,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe0,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0xdc,0x02,0x00,0x00,0x26,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xdd,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x2a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xdd,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, -0x92,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x30,0x01,0x00,0x00,0xdd,0x02,0x00,0x00, -0xa7,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x30,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x7f,0x00,0x00,0x00, -0xdd,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x38,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0xab,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x38,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0xe4,0x02,0x00,0x00, -0x39,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x35,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x7a,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x45,0x01,0x00,0x00,0x42,0x01,0x00,0x00,0x44,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x55,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x55,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00, -0x57,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x45,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x57,0x01,0x00,0x00,0x56,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x08,0x00, -0x12,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x5c,0x01,0x00,0x00,0x5b,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x4a,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x45,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x63,0x01,0x00,0x00, -0x62,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x64,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0x60,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x65,0x01,0x00,0x00, +0x06,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x8a,0x00,0x00,0x00, +0x83,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, +0x88,0x00,0x00,0x00,0x8d,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0x91,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x92,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x33,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x98,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x14,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00, +0xa7,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0xa4,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0xac,0x00,0x00,0x00,0x84,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x00,0x00,0x00, +0xae,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb1,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0xc0,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xb3,0x00,0x00,0x00,0xb2,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc2,0x00,0x00,0x00, +0xb2,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb2,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00, +0xcd,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0xa3,0x02,0x00,0x00, +0x3e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x00,0x00,0x00, +0xa3,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xb1,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd3,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbc,0x02,0x00,0x00,0xaf,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0xd6,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb8,0x02,0x00,0x00,0x9e,0x00,0x00,0x00, +0xb3,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0xd6,0x00,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa4,0x02,0x00,0x00, +0x84,0x00,0x00,0x00,0xb3,0x00,0x00,0x00,0x1f,0x02,0x00,0x00, +0xd6,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xa4,0x02,0x00,0x00,0x8e,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xda,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd4,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0xdf,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe2,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0xe6,0x00,0x00,0x00, +0xb4,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0xe8,0x00,0x00,0x00,0x37,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xed,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xeb,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0xa4,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, +0x8e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xed,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, +0xc1,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xeb,0x00,0x00,0x00, +0xdd,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0xf5,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xf3,0x00,0x00,0x00,0xf4,0x00,0x00,0x00, +0x15,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf4,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0xfd,0x00,0x00,0x00, +0xfe,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x01,0x01,0x00,0x00,0xff,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, +0xfd,0x00,0x00,0x00,0x99,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, +0x0c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x0f,0x01,0x00,0x00,0x0d,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0x10,0x01,0x00,0x00,0x11,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00,0x12,0x01,0x00,0x00, +0x11,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x01,0x00,0x00, +0x14,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x01,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x14,0x01,0x00,0x00,0x12,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x15,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x18,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, +0x18,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1c,0x01,0x00,0x00,0x1a,0x01,0x00,0x00, +0x6f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x13,0x01,0x00,0x00, +0x1d,0x01,0x00,0x00,0xfa,0x00,0x00,0x00,0x1c,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x1d,0x01,0x00,0x00,0xcb,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xf5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdf,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00,0xb4,0x02,0x00,0x00, +0x22,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x26,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0x29,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x2c,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0xa6,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x28,0x01,0x00,0x00,0x29,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2c,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x28,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x27,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x30,0x01,0x00,0x00,0xa7,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x32,0x01,0x00,0x00, +0x30,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x17,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x14,0x00,0x00,0x00, +0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x34,0x01,0x00,0x00,0x33,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0x32,0x01,0x00,0x00, +0x34,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x37,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x35,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x36,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x01,0x00,0x00,0xa4,0x02,0x00,0x00,0x79,0x00,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0x8e,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x37,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x37,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, +0x35,0x01,0x00,0x00,0x27,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x36,0x01,0x00,0x00,0xf7,0x00,0x03,0x00,0x3f,0x01,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3d,0x01,0x00,0x00, +0x3e,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x47,0x01,0x00,0x00,0x7e,0x00,0x00,0x00,0xb5,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x47,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x49,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x57,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0xaa,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, +0xbc,0x02,0x00,0x00,0x57,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00, +0x79,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0x5b,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x35,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0x4c,0x01,0x00,0x00, +0x5d,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x73,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, +0x41,0x00,0x05,0x00,0x13,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x60,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x63,0x01,0x00,0x00, +0x7e,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x63,0x01,0x00,0x00, 0x64,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x68,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x67,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0x6b,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00, -0x6d,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x68,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x6d,0x01,0x00,0x00,0x6c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x6f,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x12,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x73,0x01,0x00,0x00,0x72,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x74,0x01,0x00,0x00,0x73,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0x75,0x01,0x00,0x00, -0x4a,0x01,0x00,0x00,0x70,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x75,0x01,0x00,0x00,0x74,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x78,0x01,0x00,0x00,0x45,0x01,0x00,0x00, -0x77,0x01,0x00,0x00,0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00, -0x7a,0x01,0x00,0x00,0x52,0x01,0x00,0x00,0x35,0x00,0x00,0x00, -0x3c,0x01,0x00,0x00,0xd0,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x7b,0x01,0x00,0x00, -0x7a,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x7c,0x01,0x00,0x00,0x7b,0x01,0x00,0x00,0x41,0x00,0x05,0x00, -0x16,0x01,0x00,0x00,0x7d,0x01,0x00,0x00,0x4a,0x01,0x00,0x00, -0x78,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x7d,0x01,0x00,0x00, -0x7c,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x80,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x7f,0x01,0x00,0x00, -0x41,0x00,0x08,0x00,0x12,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, -0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x83,0x01,0x00,0x00,0x82,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x84,0x01,0x00,0x00, -0x83,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00, -0x85,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x80,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x85,0x01,0x00,0x00,0x84,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x88,0x01,0x00,0x00, -0x45,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x41,0x00,0x08,0x00, -0x12,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x52,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x3c,0x01,0x00,0x00,0xd0,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x8b,0x01,0x00,0x00,0x8a,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0x8c,0x01,0x00,0x00,0x8b,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0x8d,0x01,0x00,0x00, -0x4a,0x01,0x00,0x00,0x88,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0x8d,0x01,0x00,0x00,0x8c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00,0xdd,0x02,0x00,0x00, -0x90,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2a,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x96,0x01,0x00,0x00, -0xe0,0x02,0x00,0x00,0x94,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0xe4,0x02,0x00,0x00, -0x97,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x9b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x9b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xe6,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x2c,0x01,0x00,0x00,0x45,0x02,0x00,0x00,0x9e,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0xe6,0x02,0x00,0x00,0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x9d,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xa1,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, -0x9d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9c,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xea,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, -0xcf,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xea,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xa5,0x01,0x00,0x00, -0xa6,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xa9,0x01,0x00,0x00,0xa4,0x01,0x00,0x00,0xa5,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa4,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x67,0x01,0x00,0x00,0x65,0x01,0x00,0x00,0x79,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x13,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x44,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, +0x68,0x01,0x00,0x00,0xcb,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x3f,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3f,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x29,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x6d,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x6b,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x26,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x28,0x01,0x00,0x00,0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00,0xb8,0x02,0x00,0x00, +0x6f,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0xbc,0x02,0x00,0x00,0x72,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x76,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xbe,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x1d,0x02,0x00,0x00,0x79,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x7c,0x01,0x00,0x00,0xbe,0x02,0x00,0x00, +0x6d,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x78,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x7c,0x01,0x00,0x00,0x77,0x01,0x00,0x00,0x78,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x77,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7e,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc2,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x77,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, +0x81,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x84,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x80,0x01,0x00,0x00,0x81,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x84,0x01,0x00,0x00, +0x7f,0x01,0x00,0x00,0x80,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x86,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x7f,0x01,0x00,0x00,0xa7,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x8c,0x01,0x00,0x00, +0xd4,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x88,0x01,0x00,0x00,0x87,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x8c,0x01,0x00,0x00,0x87,0x01,0x00,0x00, +0x88,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x01,0x00,0x00,0x92,0x01,0x00,0x00, +0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x96,0x01,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, +0xc2,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x96,0x01,0x00,0x00, +0x98,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x9b,0x01,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00, +0x99,0x01,0x00,0x00,0x9b,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x01,0x00,0x00,0x9c,0x01,0x00,0x00, +0xd4,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa0,0x01,0x00,0x00,0x9e,0x01,0x00,0x00,0x9f,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa2,0x01,0x00,0x00, +0xa0,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x13,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, +0xa2,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xa4,0x01,0x00,0x00,0xa3,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xa5,0x01,0x00,0x00,0x90,0x01,0x00,0x00, +0x94,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xa5,0x01,0x00,0x00, +0xa4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa7,0x01,0x00,0x00,0xd4,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x86,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x88,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x81,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x81,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xa9,0x01,0x00,0x00,0xc2,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x80,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, 0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xab,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xfc,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xa4,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0xac,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xb1,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00,0xac,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc3,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x80,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, +0xae,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xb1,0x01,0x00,0x00,0xc3,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xad,0x01,0x00,0x00,0xae,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xb1,0x01,0x00,0x00, 0xac,0x01,0x00,0x00,0xad,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xac,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb7,0x01,0x00,0x00,0xea,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, -0xb7,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbb,0x01,0x00,0x00,0x56,0x00,0x00,0x00, -0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbd,0x01,0x00,0x00,0xea,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbe,0x01,0x00,0x00, -0xbb,0x01,0x00,0x00,0xbd,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00,0x65,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc1,0x01,0x00,0x00,0xbe,0x01,0x00,0x00,0xc0,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00, -0xc1,0x01,0x00,0x00,0xfc,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc5,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0xc4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc7,0x01,0x00,0x00,0xc5,0x01,0x00,0x00,0xe6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0xfc,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0xc9,0x01,0x00,0x00,0xc8,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xca,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0xb5,0x01,0x00,0x00,0xb9,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xcb,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0xfc,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00, +0xac,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xd1,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xac,0x01,0x00,0x00,0xd5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xb9,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xb5,0x01,0x00,0x00,0xb4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0xb4,0x01,0x00,0x00, +0xb5,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x01,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc1,0x01,0x00,0x00,0xbf,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc3,0x01,0x00,0x00,0x5a,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, +0xc3,0x02,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, +0xc6,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xc9,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xca,0x01,0x00,0x00, +0xc7,0x01,0x00,0x00,0xc9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xcc,0x01,0x00,0x00,0xca,0x01,0x00,0x00, +0xd1,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xce,0x01,0x00,0x00,0xcc,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd0,0x01,0x00,0x00, +0xce,0x01,0x00,0x00,0xbe,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x13,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x44,0x01,0x00,0x00, +0xd0,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0xd2,0x01,0x00,0x00,0xd1,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0xd3,0x01,0x00,0x00,0xbd,0x01,0x00,0x00, +0xc1,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0xd3,0x01,0x00,0x00, +0xd2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xd5,0x01,0x00,0x00,0xd1,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xb3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xb5,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xae,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xae,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xc3,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00, 0xf8,0x00,0x02,0x00,0xad,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa6,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, -0xea,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa5,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xeb,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xa5,0x01,0x00,0x00, -0xfd,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xd7,0x01,0x00,0x00,0xeb,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xd3,0x01,0x00,0x00, -0xd4,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xd7,0x01,0x00,0x00,0xd2,0x01,0x00,0x00,0xd3,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd2,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, 0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf9,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xd2,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, -0xda,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xdf,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00,0xda,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc4,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xad,0x01,0x00,0x00,0x1b,0x02,0x00,0x00, +0xdc,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xdf,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0xbe,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xdb,0x01,0x00,0x00,0xdc,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdf,0x01,0x00,0x00, 0xda,0x01,0x00,0x00,0xdb,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xda,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe5,0x01,0x00,0x00,0xeb,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, -0xe5,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x01,0x00,0x00,0x5a,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xeb,0x02,0x00,0x00,0xeb,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xed,0x01,0x00,0x00, -0xe9,0x01,0x00,0x00,0xec,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0x69,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf0,0x01,0x00,0x00,0xed,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xf9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf4,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, -0xf3,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xf6,0x01,0x00,0x00,0xf4,0x01,0x00,0x00,0xe6,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x16,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, -0x4a,0x01,0x00,0x00,0xf6,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, -0xf7,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0xf7,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0xca,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, -0xe3,0x01,0x00,0x00,0xe7,0x01,0x00,0x00,0x3e,0x00,0x03,0x00, -0xf9,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00,0xf9,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, +0xda,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0xda,0x01,0x00,0x00,0x19,0x02,0x00,0x00,0xe4,0x01,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0xe7,0x01,0x00,0x00, +0xc8,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0xe3,0x01,0x00,0x00,0xe4,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0xe7,0x01,0x00,0x00,0xe2,0x01,0x00,0x00, +0xe3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe2,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe9,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xca,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xe2,0x01,0x00,0x00, +0x17,0x02,0x00,0x00,0xec,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0xef,0x01,0x00,0x00,0xca,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xeb,0x01,0x00,0x00, +0xec,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0xef,0x01,0x00,0x00,0xea,0x01,0x00,0x00,0xeb,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xea,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xf1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xcc,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0xea,0x01,0x00,0x00,0x15,0x02,0x00,0x00, +0xf2,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0xf7,0x01,0x00,0x00,0xcc,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0xf3,0x01,0x00,0x00,0xf2,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xf7,0x01,0x00,0x00, +0xf2,0x01,0x00,0x00,0xf3,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xf2,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xf9,0x01,0x00,0x00,0xc4,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfb,0x01,0x00,0x00, +0xf9,0x01,0x00,0x00,0xca,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00, +0xfc,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xff,0x01,0x00,0x00,0xc8,0x02,0x00,0x00,0x63,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x02,0x00,0x00, +0xfd,0x01,0x00,0x00,0xff,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x00,0x02,0x00,0x00, +0xcc,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x06,0x02,0x00,0x00,0xff,0x01,0x00,0x00,0xcc,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x07,0x02,0x00,0x00, +0x90,0x01,0x00,0x00,0x06,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x08,0x02,0x00,0x00,0x07,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x0d,0x02,0x00,0x00, +0xbd,0x01,0x00,0x00,0xfb,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0xcc,0x00,0x00,0x00,0x10,0x02,0x00,0x00, +0xc9,0x00,0x00,0x00,0x02,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0xc3,0x00,0x00,0x00,0x11,0x02,0x00,0x00,0x10,0x02,0x00,0x00, +0x0c,0x00,0x08,0x00,0xc3,0x00,0x00,0x00,0x12,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x08,0x02,0x00,0x00, +0x0e,0x02,0x00,0x00,0x11,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x10,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x15,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xf1,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xf3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0xec,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xec,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x17,0x02,0x00,0x00, +0xca,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xe9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xeb,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe4,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x19,0x02,0x00,0x00,0xc8,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0xe3,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdc,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0xdc,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1b,0x02,0x00,0x00,0xc4,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00, 0xf8,0x00,0x02,0x00,0xdb,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd4,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd4,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00, -0xeb,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xd1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd3,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xff,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xec,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd3,0x01,0x00,0x00, -0x43,0x02,0x00,0x00,0x02,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0xec,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x01,0x02,0x00,0x00, -0x02,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x05,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x01,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x00,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x07,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xf0,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x41,0x02,0x00,0x00, -0x0a,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x0d,0x02,0x00,0x00,0xf0,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x09,0x02,0x00,0x00,0x0a,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x0d,0x02,0x00,0x00, -0x08,0x02,0x00,0x00,0x09,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x08,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x0f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x0f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xf2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x08,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0x12,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x15,0x02,0x00,0x00, -0xf2,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x11,0x02,0x00,0x00,0x12,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x15,0x02,0x00,0x00,0x10,0x02,0x00,0x00, -0x11,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x10,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x17,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xf4,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x10,0x02,0x00,0x00, -0x3d,0x02,0x00,0x00,0x18,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x1d,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x19,0x02,0x00,0x00, -0x18,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x1d,0x02,0x00,0x00,0x18,0x02,0x00,0x00,0x19,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x18,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0xec,0x02,0x00,0x00, -0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x1f,0x02,0x00,0x00,0xf2,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x23,0x02,0x00,0x00, -0x21,0x02,0x00,0x00,0x22,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x02,0x00,0x00,0xf0,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x02,0x00,0x00,0x23,0x02,0x00,0x00,0x25,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x28,0x02,0x00,0x00, -0x26,0x02,0x00,0x00,0xf4,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x2c,0x02,0x00,0x00,0x25,0x02,0x00,0x00, -0xf4,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xca,0x01,0x00,0x00, -0x2d,0x02,0x00,0x00,0xb5,0x01,0x00,0x00,0x2c,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x2d,0x02,0x00,0x00,0x73,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x2f,0x02,0x00,0x00,0x2e,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xca,0x01,0x00,0x00,0x34,0x02,0x00,0x00,0xe3,0x01,0x00,0x00, -0x21,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xf7,0x00,0x00,0x00, -0x35,0x02,0x00,0x00,0x34,0x02,0x00,0x00,0x73,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x36,0x02,0x00,0x00,0x35,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00,0x38,0x02,0x00,0x00, -0xca,0x00,0x00,0x00,0x28,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x38,0x02,0x00,0x00, -0x0c,0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x3a,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x2f,0x02,0x00,0x00, -0x36,0x02,0x00,0x00,0x39,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0x38,0x02,0x00,0x00,0x3a,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0xf4,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x19,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x12,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x12,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3f,0x02,0x00,0x00, -0xf2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x0f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x11,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x0a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x0a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x02,0x00,0x00,0xf0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x07,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x09,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x02,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x02,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x43,0x02,0x00,0x00,0xec,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xff,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x01,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9e,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x45,0x02,0x00,0x00, -0xe6,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x9d,0x01,0x00,0x00, +0x79,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x79,0x01,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1d,0x02,0x00,0x00, +0xbe,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x76,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x01,0x00,0x00, 0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x93,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x47,0x02,0x00,0x00,0xcc,0x02,0x00,0x00, -0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x56,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd6,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x1f,0x02,0x00,0x00,0xa4,0x02,0x00,0x00, +0x6d,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xd3,0x00,0x00,0x00, +0xf8,0x00,0x02,0x00,0xd5,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x24,0x02,0x00,0x00,0x56,0x00,0x00,0x00, 0x54,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4d,0x02,0x00,0x00,0x97,0x00,0x00,0x00,0x4c,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x02,0x00,0x00, -0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0xa8,0x00,0x00,0x00, -0x52,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x57,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x58,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x59,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x58,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5d,0x02,0x00,0x00, -0x48,0x00,0x00,0x00,0x58,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x5f,0x02,0x00,0x00,0x5e,0x02,0x00,0x00, +0x25,0x02,0x00,0x00,0x96,0x00,0x00,0x00,0x24,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2a,0x02,0x00,0x00, +0x5a,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, +0x2a,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, +0x2f,0x02,0x00,0x00,0x14,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0x2f,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x31,0x02,0x00,0x00,0x0f,0x00,0x00,0x00,0x30,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00, +0x48,0x00,0x00,0x00,0x30,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0x0d,0x00,0x00,0x00,0x37,0x02,0x00,0x00,0x36,0x02,0x00,0x00, 0x0c,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x02,0x00,0x00,0x5f,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x61,0x02,0x00,0x00,0x5d,0x02,0x00,0x00, -0x60,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x62,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0x61,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x64,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xcd,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, -0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x66,0x02,0x00,0x00, -0x67,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x6a,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x66,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6c,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xce,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x65,0x02,0x00,0x00,0xc8,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x72,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0x61,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x6e,0x02,0x00,0x00,0x6f,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x72,0x02,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x6d,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x76,0x02,0x00,0x00,0xce,0x02,0x00,0x00,0x62,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00, -0x4d,0x02,0x00,0x00,0x76,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x79,0x02,0x00,0x00,0x65,0x00,0x00,0x00, +0x38,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x39,0x02,0x00,0x00,0x35,0x02,0x00,0x00, +0x38,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x3a,0x02,0x00,0x00,0x31,0x02,0x00,0x00,0x39,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0xa5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0x3f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x42,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xbe,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x3e,0x02,0x00,0x00, +0x3f,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, +0x42,0x02,0x00,0x00,0x3d,0x02,0x00,0x00,0x3e,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x3d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x44,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa6,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x3d,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, +0x47,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x4a,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x61,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x46,0x02,0x00,0x00,0x47,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x4a,0x02,0x00,0x00, +0x45,0x02,0x00,0x00,0x46,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x45,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x4e,0x02,0x00,0x00,0xa6,0x02,0x00,0x00,0x62,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, +0x25,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x51,0x02,0x00,0x00,0x65,0x00,0x00,0x00, 0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7e,0x02,0x00,0x00, -0xcd,0x02,0x00,0x00,0xeb,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x7e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x82,0x02,0x00,0x00, -0x7f,0x02,0x00,0x00,0x81,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x84,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x84,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xd0,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x6d,0x02,0x00,0x00,0xc6,0x02,0x00,0x00, -0x87,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x8a,0x02,0x00,0x00,0xd0,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x86,0x02,0x00,0x00,0x87,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x8a,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x85,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x8c,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x8c,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xd2,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0xc4,0x02,0x00,0x00,0x8f,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x92,0x02,0x00,0x00, -0xd2,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x8e,0x02,0x00,0x00,0x8f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x92,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x8d,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x95,0x02,0x00,0x00, -0x7a,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0x95,0x02,0x00,0x00, -0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x9a,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x98,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x99,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9d,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x9e,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x9d,0x02,0x00,0x00,0x9f,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x9a,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x9a,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0xc2,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x98,0x02,0x00,0x00,0x8d,0x02,0x00,0x00,0xa0,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0xa3,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xa1,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0xa2,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xab,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xad,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0xac,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xae,0x02,0x00,0x00,0xad,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xaf,0x02,0x00,0x00, -0xab,0x02,0x00,0x00,0xae,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb0,0x02,0x00,0x00,0x62,0x02,0x00,0x00, -0xaf,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb2,0x02,0x00,0x00,0xb0,0x02,0x00,0x00,0x7a,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb4,0x02,0x00,0x00, -0xb2,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xb6,0x02,0x00,0x00,0xcd,0x02,0x00,0x00, -0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb6,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xba,0x02,0x00,0x00, -0xb8,0x02,0x00,0x00,0xb9,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x02,0x00,0x00,0xce,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x51,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, +0xa5,0x02,0x00,0x00,0xc5,0x01,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x2b,0x02,0x00,0x00, +0x56,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x59,0x02,0x00,0x00,0x69,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, +0x57,0x02,0x00,0x00,0x59,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa8,0x02,0x00,0x00, +0x3f,0x00,0x00,0x00,0x45,0x02,0x00,0x00,0x9e,0x02,0x00,0x00, +0x5f,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00, +0x62,0x02,0x00,0x00,0xa8,0x02,0x00,0x00,0xbb,0x00,0x00,0x00, +0xf6,0x00,0x04,0x00,0x5e,0x02,0x00,0x00,0x5f,0x02,0x00,0x00, +0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x62,0x02,0x00,0x00, +0x5d,0x02,0x00,0x00,0x5e,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x5d,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x64,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xaa,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, +0x5d,0x02,0x00,0x00,0x9c,0x02,0x00,0x00,0x67,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x6a,0x02,0x00,0x00, +0xaa,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, +0x66,0x02,0x00,0x00,0x67,0x02,0x00,0x00,0x01,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x6a,0x02,0x00,0x00,0x65,0x02,0x00,0x00, +0x66,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x65,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x02,0x00,0x00, +0x52,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, +0xc1,0x00,0x00,0x00,0x70,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, +0x37,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x72,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x70,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x71,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x75,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x76,0x02,0x00,0x00, +0x14,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x77,0x02,0x00,0x00,0x76,0x02,0x00,0x00, +0xb0,0x00,0x05,0x00,0xc1,0x00,0x00,0x00,0x78,0x02,0x00,0x00, +0x75,0x02,0x00,0x00,0x77,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x72,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x72,0x02,0x00,0x00, +0xf5,0x00,0x07,0x00,0xc1,0x00,0x00,0x00,0x79,0x02,0x00,0x00, +0x70,0x02,0x00,0x00,0x65,0x02,0x00,0x00,0x78,0x02,0x00,0x00, +0x71,0x02,0x00,0x00,0xf7,0x00,0x03,0x00,0x7b,0x02,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x02,0x00,0x00, +0x7a,0x02,0x00,0x00,0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x7a,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x83,0x02,0x00,0x00,0x5a,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x85,0x02,0x00,0x00, +0x14,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x86,0x02,0x00,0x00,0x85,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00, +0x83,0x02,0x00,0x00,0x86,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x88,0x02,0x00,0x00,0x3a,0x02,0x00,0x00, +0x87,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x8a,0x02,0x00,0x00,0x88,0x02,0x00,0x00,0x52,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8c,0x02,0x00,0x00, +0x8a,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00,0xa5,0x02,0x00,0x00, +0xbb,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x90,0x02,0x00,0x00,0x8e,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00, +0x90,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x84,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0xa6,0x02,0x00,0x00, 0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbd,0x02,0x00,0x00,0xba,0x02,0x00,0x00,0xbc,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00, -0xbd,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0xca,0x00,0x00,0x00, -0xbf,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0xc1,0x02,0x00,0x00,0xc0,0x02,0x00,0x00,0x41,0x00,0x06,0x00, -0x12,0x01,0x00,0x00,0xc2,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, -0x35,0x00,0x00,0x00,0xb4,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, -0xc2,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0xa3,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0xa3,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc4,0x02,0x00,0x00,0xd2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x8c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x8e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x87,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x87,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc6,0x02,0x00,0x00,0xd0,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x84,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x86,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc8,0x02,0x00,0x00, -0xce,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x6c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0x92,0x02,0x00,0x00,0x94,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x97,0x02,0x00,0x00, +0x95,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0x41,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x98,0x02,0x00,0x00,0xc9,0x00,0x00,0x00, +0x97,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, +0x99,0x02,0x00,0x00,0x98,0x02,0x00,0x00,0x41,0x00,0x06,0x00, +0x10,0x01,0x00,0x00,0x9a,0x02,0x00,0x00,0x7f,0x02,0x00,0x00, +0x35,0x00,0x00,0x00,0x8c,0x02,0x00,0x00,0x3e,0x00,0x03,0x00, +0x9a,0x02,0x00,0x00,0x99,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x7b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x7b,0x02,0x00,0x00, 0xf9,0x00,0x02,0x00,0x67,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, 0x67,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xca,0x02,0x00,0x00,0xcd,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, +0x9c,0x02,0x00,0x00,0xaa,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, 0xf9,0x00,0x02,0x00,0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x66,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x66,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x5f,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x9e,0x02,0x00,0x00,0xa8,0x02,0x00,0x00, +0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00, +0xf8,0x00,0x02,0x00,0x5e,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, +0x47,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x47,0x02,0x00,0x00, +0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, +0xa6,0x02,0x00,0x00,0xcf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0x44,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x46,0x02,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0xa2,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0xcf,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, +0x3e,0x02,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t matmul_f32_aligned_len = 10932; - -unsigned char matmul_f32_aligned_fp32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0xd2,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0f,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x4d,0x00,0x00,0x00,0xfb,0x00,0x00,0x00,0x06,0x01,0x00,0x00, -0x47,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x33,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x0b,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00,0x0b,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x12,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x12,0x00,0x00,0x00, -0x0d,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x12,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x38,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x3e,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4d,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x50,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x54,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x61,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x63,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x6d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xa7,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xb9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xbc,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x03,0x01,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x04,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x04,0x01,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x06,0x01,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x1f,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x20,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x4b,0x01,0x00,0x00, -0x06,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x04,0x00, -0x4c,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x4c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x4c,0x01,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x4e,0x01,0x00,0x00,0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x4e,0x01,0x00,0x00,0x21,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x33,0x02,0x00,0x00, -0x0b,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x79,0x02,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x7a,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x7a,0x02,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x7a,0x02,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x7c,0x02,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x7c,0x02,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x13,0x00,0x02,0x00, -0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x17,0x00,0x04,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x1e,0x00,0x10,0x00,0x12,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x13,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x12,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x13,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x16,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x17,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x35,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x40,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00, -0x3e,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x3f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x0a,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x59,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x38,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x64,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x62,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6e,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x79,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x78,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x82,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00,0x92,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x98,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x32,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, -0x61,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0xba,0x00,0x00,0x00, -0xbe,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc0,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x84,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0xc2,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc6,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0xc6,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0xc8,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0xc7,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0xc4,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xf9,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xfa,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0xf9,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xfa,0x00,0x00,0x00, -0xfb,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xff,0x00,0x00,0x00,0x80,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x03,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x04,0x01,0x00,0x00,0x03,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x05,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x04,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x05,0x01,0x00,0x00,0x06,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x11,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x14,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x1a,0x01,0x00,0x00, -0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x32,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1f,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x33,0x00,0x06,0x00,0x09,0x00,0x00,0x00, -0x20,0x01,0x00,0x00,0x1f,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x21,0x01,0x00,0x00,0x51,0x00,0x00,0x00,0x20,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0x22,0x01,0x00,0x00, -0x6d,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x43,0x01,0x00,0x00,0x1c,0x00,0x04,0x00,0x45,0x01,0x00,0x00, -0xc4,0x00,0x00,0x00,0x44,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x46,0x01,0x00,0x00,0x04,0x00,0x00,0x00,0x45,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x46,0x01,0x00,0x00,0x47,0x01,0x00,0x00, -0x04,0x00,0x00,0x00,0x17,0x00,0x04,0x00,0x4a,0x01,0x00,0x00, -0xc4,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x1e,0x00,0x03,0x00, -0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00,0x20,0x00,0x04,0x00, -0x4d,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x4c,0x01,0x00,0x00, -0x3b,0x00,0x04,0x00,0x4d,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x60,0x01,0x00,0x00,0x03,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x51,0x00,0x00,0x00, -0x20,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0x66,0x01,0x00,0x00,0x78,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00,0x86,0x00,0x00,0x00, -0x67,0x01,0x00,0x00,0x6d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x08,0x01,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6c,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x6f,0x01,0x00,0x00, -0x86,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8a,0x01,0x00,0x00, -0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x8b,0x01,0x00,0x00,0xc4,0x00,0x00,0x00, -0x8a,0x01,0x00,0x00,0x20,0x00,0x04,0x00,0x8c,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x8b,0x01,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x9c,0x01,0x00,0x00,0x80,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0xb7,0x01,0x00,0x00,0x84,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0xb8,0x01,0x00,0x00,0xc4,0x00,0x00,0x00,0xb7,0x01,0x00,0x00, -0x20,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0xb8,0x01,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xc2,0x01,0x00,0x00,0x86,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xca,0x01,0x00,0x00,0x80,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0xf9,0x01,0x00,0x00,0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00, -0x63,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x15,0x00,0x00,0x00, -0x2b,0x02,0x00,0x00,0x0d,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x0a,0x00,0x00,0x00,0x33,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x79,0x02,0x00,0x00,0xc4,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x7a,0x02,0x00,0x00,0x79,0x02,0x00,0x00, -0x20,0x00,0x04,0x00,0x7b,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x7a,0x02,0x00,0x00,0x3b,0x00,0x04,0x00,0x7b,0x02,0x00,0x00, -0x7c,0x02,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x15,0x00,0x00,0x00,0x81,0x02,0x00,0x00,0x05,0x00,0x00,0x00, -0x34,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x8e,0x02,0x00,0x00, -0x84,0x00,0x00,0x00,0x61,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xc9,0x00,0x00,0x00, -0xca,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x8c,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x07,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0xb9,0x01,0x00,0x00,0xba,0x01,0x00,0x00, -0x07,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x16,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1a,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x0f,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0x24,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x2f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x25,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x31,0x00,0x00,0x00, -0x2b,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x35,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0x36,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x37,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x82,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x39,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x3b,0x00,0x00,0x00, -0x38,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x3e,0x00,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x40,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x43,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x48,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x3c,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x3e,0x00,0x00,0x00, -0x3a,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x00,0x00,0x00,0x4a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x0d,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x4d,0x00,0x00,0x00, -0x3f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x4e,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x56,0x00,0x00,0x00,0x51,0x00,0x00,0x00,0x55,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x00,0x00,0x00, -0x51,0x00,0x00,0x00,0x59,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x50,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x65,0x00,0x00,0x00,0x5e,0x00,0x00,0x00,0x64,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x5e,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x6e,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0x4f,0x00,0x00,0x00,0x73,0x00,0x00,0x00, -0x89,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x7a,0x00,0x00,0x00, -0x4f,0x00,0x00,0x00,0x79,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x7f,0x00,0x00,0x00,0x4f,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x82,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x84,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x89,0x00,0x00,0x00,0x88,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x48,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x8b,0x00,0x00,0x00, -0x84,0x00,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x26,0x00,0x00,0x00, -0x89,0x00,0x00,0x00,0x8e,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0x92,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x38,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x99,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x95,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x00,0x00,0x00, -0x9e,0x00,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x14,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x0f,0x00,0x00,0x00, -0xa4,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0x4b,0x00,0x00,0x00,0xa7,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x14,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xac,0x00,0x00,0x00, -0xa8,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0xac,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0xad,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xaf,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xc3,0x00,0x00,0x00,0xa0,0x02,0x00,0x00,0xc1,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xb4,0x00,0x00,0x00,0xb3,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xc3,0x00,0x00,0x00, -0xb3,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0xca,0x00,0x00,0x00,0xa0,0x02,0x00,0x00, -0x3e,0x00,0x03,0x00,0xce,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xa0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb2,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xb4,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd4,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xb9,0x02,0x00,0x00,0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, -0x71,0x01,0x00,0x00,0xd7,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb5,0x02,0x00,0x00,0x9f,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa1,0x02,0x00,0x00, -0x85,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x1c,0x02,0x00,0x00, -0xd7,0x00,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xa1,0x02,0x00,0x00,0x8f,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xd6,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xdb,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd5,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdd,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0xe0,0x00,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xe3,0x00,0x00,0x00, -0xb1,0x02,0x00,0x00,0x38,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xdf,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xe3,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0xdf,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xde,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe7,0x00,0x00,0x00, -0xb1,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xec,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xec,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xee,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xed,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0xa1,0x02,0x00,0x00,0x6f,0x00,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, -0x8f,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x00,0x00,0x00,0xf5,0x00,0x07,0x00, -0xc2,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0xec,0x00,0x00,0x00, -0xde,0x00,0x00,0x00,0xf3,0x00,0x00,0x00,0xed,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0xf6,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf4,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, -0x16,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xf5,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfe,0x00,0x00,0x00, -0x74,0x00,0x00,0x00,0xb1,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0xfe,0x00,0x00,0x00, -0xff,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x02,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, -0xfe,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0e,0x01,0x00,0x00,0xb5,0x02,0x00,0x00, -0x0d,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x10,0x01,0x00,0x00,0x0e,0x01,0x00,0x00,0x6f,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x06,0x01,0x00,0x00,0x35,0x00,0x00,0x00,0x10,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x13,0x01,0x00,0x00, -0x12,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, -0x15,0x01,0x00,0x00,0xfb,0x00,0x00,0x00,0x02,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x15,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x16,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x19,0x01,0x00,0x00,0x74,0x00,0x00,0x00,0xb1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x1b,0x01,0x00,0x00, -0x19,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x1b,0x01,0x00,0x00, -0x6f,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0xfb,0x00,0x00,0x00,0x1d,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x1e,0x01,0x00,0x00,0xcc,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xf6,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf6,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe0,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe0,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x25,0x01,0x00,0x00,0xb1,0x02,0x00,0x00, -0x23,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xdd,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x27,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x27,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x2d,0x01,0x00,0x00,0xb2,0x02,0x00,0x00,0xa7,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x2d,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x28,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x32,0x01,0x00,0x00,0x7f,0x00,0x00,0x00,0xb2,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x32,0x01,0x00,0x00,0xab,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x36,0x01,0x00,0x00,0x35,0x01,0x00,0x00, -0x78,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x37,0x01,0x00,0x00,0xb9,0x02,0x00,0x00,0x36,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0x37,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x3f,0x01,0x00,0x00,0x32,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x41,0x01,0x00,0x00,0x7a,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, -0x3f,0x01,0x00,0x00,0x41,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0x11,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x51,0x01,0x00,0x00, -0x50,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x42,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x51,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x3a,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x11,0x01,0x00,0x00,0x56,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x3a,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0x56,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x54,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x58,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x0c,0x00,0x00,0x00,0x41,0x00,0x07,0x00, -0x11,0x01,0x00,0x00,0x5c,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x5d,0x01,0x00,0x00, -0x5c,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, -0x5e,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x5e,0x01,0x00,0x00,0x5d,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x61,0x01,0x00,0x00, -0x42,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x41,0x00,0x07,0x00, -0x11,0x01,0x00,0x00,0x63,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x35,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x60,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x64,0x01,0x00,0x00, -0x63,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, -0x65,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0x61,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, -0xb2,0x02,0x00,0x00,0x68,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x27,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x29,0x01,0x00,0x00, -0xe0,0x00,0x04,0x00,0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x6b,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x6e,0x01,0x00,0x00,0xb5,0x02,0x00,0x00,0x6c,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x71,0x01,0x00,0x00, -0xb9,0x02,0x00,0x00,0x6f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x73,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x73,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xbb,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x1a,0x02,0x00,0x00, -0x76,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x79,0x01,0x00,0x00,0xbb,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x75,0x01,0x00,0x00,0x76,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x79,0x01,0x00,0x00, -0x74,0x01,0x00,0x00,0x75,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x74,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x7b,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x7b,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xbf,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x74,0x01,0x00,0x00,0xa6,0x01,0x00,0x00,0x7e,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x81,0x01,0x00,0x00, -0xbf,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x7d,0x01,0x00,0x00,0x7e,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x81,0x01,0x00,0x00,0x7c,0x01,0x00,0x00, -0x7d,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x7c,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x83,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x83,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xd1,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x7c,0x01,0x00,0x00, -0xa4,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x89,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x85,0x01,0x00,0x00, -0x84,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x89,0x01,0x00,0x00,0x84,0x01,0x00,0x00,0x85,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x84,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x01,0x00,0x00,0x8f,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x93,0x01,0x00,0x00, -0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x95,0x01,0x00,0x00,0xbf,0x02,0x00,0x00, -0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x96,0x01,0x00,0x00,0x93,0x01,0x00,0x00,0x95,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x98,0x01,0x00,0x00, -0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x99,0x01,0x00,0x00,0x96,0x01,0x00,0x00, -0x98,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x01,0x00,0x00,0x99,0x01,0x00,0x00,0xd1,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9d,0x01,0x00,0x00, -0x9b,0x01,0x00,0x00,0x9c,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9f,0x01,0x00,0x00,0x9d,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0xfb,0x00,0x00,0x00,0x9f,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xa1,0x01,0x00,0x00, -0xa0,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xa2,0x01,0x00,0x00,0x8d,0x01,0x00,0x00,0x91,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa2,0x01,0x00,0x00,0xa1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xa4,0x01,0x00,0x00, -0xd1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x83,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x85,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7e,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xa6,0x01,0x00,0x00,0xbf,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x7b,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x7d,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc0,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x7d,0x01,0x00,0x00,0xd4,0x01,0x00,0x00,0xab,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xae,0x01,0x00,0x00, -0xc0,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xaa,0x01,0x00,0x00,0xab,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xae,0x01,0x00,0x00,0xa9,0x01,0x00,0x00, -0xaa,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xa9,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xb0,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xce,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xa9,0x01,0x00,0x00, -0xd2,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xb6,0x01,0x00,0x00,0xce,0x02,0x00,0x00, -0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xb2,0x01,0x00,0x00, -0xb1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xb6,0x01,0x00,0x00,0xb1,0x01,0x00,0x00,0xb2,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xb1,0x01,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xbc,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xbe,0x01,0x00,0x00,0xbc,0x01,0x00,0x00,0xce,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc0,0x01,0x00,0x00, -0x5a,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc3,0x01,0x00,0x00,0xc0,0x02,0x00,0x00, -0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc4,0x01,0x00,0x00,0xc0,0x01,0x00,0x00,0xc3,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xc6,0x01,0x00,0x00, -0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xc7,0x01,0x00,0x00,0xc4,0x01,0x00,0x00, -0xc6,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xc9,0x01,0x00,0x00,0xc7,0x01,0x00,0x00,0xce,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xcb,0x01,0x00,0x00, -0xc9,0x01,0x00,0x00,0xca,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xcd,0x01,0x00,0x00,0xcb,0x01,0x00,0x00, -0xbb,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x14,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0x47,0x01,0x00,0x00,0xcd,0x01,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0xcf,0x01,0x00,0x00, -0xce,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0xd0,0x01,0x00,0x00,0xba,0x01,0x00,0x00,0xbe,0x01,0x00,0x00, -0x3e,0x00,0x03,0x00,0xd0,0x01,0x00,0x00,0xcf,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xd2,0x01,0x00,0x00, -0xce,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xb0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xb2,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xab,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xab,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd4,0x01,0x00,0x00,0xc0,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xa8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xaa,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc1,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xaa,0x01,0x00,0x00,0x18,0x02,0x00,0x00,0xd9,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xdc,0x01,0x00,0x00, -0xc1,0x02,0x00,0x00,0xbf,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xd8,0x01,0x00,0x00,0xd9,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xdc,0x01,0x00,0x00,0xd7,0x01,0x00,0x00, -0xd8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xd7,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xc5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0xd7,0x01,0x00,0x00, -0x16,0x02,0x00,0x00,0xe1,0x01,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0xe4,0x01,0x00,0x00,0xc5,0x02,0x00,0x00, -0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0xe0,0x01,0x00,0x00, -0xe1,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0xe4,0x01,0x00,0x00,0xdf,0x01,0x00,0x00,0xe0,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xdf,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xc7,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xdf,0x01,0x00,0x00,0x14,0x02,0x00,0x00, -0xe9,0x01,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0xec,0x01,0x00,0x00,0xc7,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0xe8,0x01,0x00,0x00,0xe9,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xec,0x01,0x00,0x00, -0xe7,0x01,0x00,0x00,0xe8,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xe7,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xc9,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0xe7,0x01,0x00,0x00,0x12,0x02,0x00,0x00,0xef,0x01,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0xf4,0x01,0x00,0x00, -0xc9,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0xf0,0x01,0x00,0x00,0xef,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0xf4,0x01,0x00,0x00,0xef,0x01,0x00,0x00, -0xf0,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xef,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf6,0x01,0x00,0x00, -0xc1,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xf8,0x01,0x00,0x00,0xf6,0x01,0x00,0x00, -0xc7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xfa,0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0xf9,0x01,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xfc,0x01,0x00,0x00, -0xc5,0x02,0x00,0x00,0x63,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xfd,0x01,0x00,0x00,0xfa,0x01,0x00,0x00, -0xfc,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0xfd,0x01,0x00,0x00,0xc9,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x03,0x02,0x00,0x00, -0xfc,0x01,0x00,0x00,0xc9,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0x04,0x02,0x00,0x00,0x8d,0x01,0x00,0x00, -0x03,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x05,0x02,0x00,0x00,0x04,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0x0a,0x02,0x00,0x00,0xba,0x01,0x00,0x00, -0xf8,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x0b,0x02,0x00,0x00,0x0a,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0xcd,0x00,0x00,0x00,0x0d,0x02,0x00,0x00,0xca,0x00,0x00,0x00, -0xff,0x01,0x00,0x00,0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00, -0x0e,0x02,0x00,0x00,0x0d,0x02,0x00,0x00,0x0c,0x00,0x08,0x00, -0xc4,0x00,0x00,0x00,0x0f,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x05,0x02,0x00,0x00,0x0b,0x02,0x00,0x00, -0x0e,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x0d,0x02,0x00,0x00, -0x0f,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x12,0x02,0x00,0x00,0xc9,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xee,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xf0,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0xe9,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe9,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x14,0x02,0x00,0x00,0xc7,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0xe6,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0xe8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0xe1,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe1,0x01,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x16,0x02,0x00,0x00, -0xc5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0xde,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0xe0,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd9,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd9,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x18,0x02,0x00,0x00,0xc1,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd6,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd8,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x76,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x76,0x01,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x1a,0x02,0x00,0x00,0xbb,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x73,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x75,0x01,0x00,0x00,0xe0,0x00,0x04,0x00, -0x0c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd7,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd7,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x1c,0x02,0x00,0x00,0xa1,0x02,0x00,0x00,0x6d,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0xd4,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0xd6,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x21,0x02,0x00,0x00,0x56,0x00,0x00,0x00,0x54,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x22,0x02,0x00,0x00, -0x97,0x00,0x00,0x00,0x21,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x27,0x02,0x00,0x00,0x5a,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x28,0x02,0x00,0x00,0xa8,0x00,0x00,0x00,0x27,0x02,0x00,0x00, -0x41,0x00,0x05,0x00,0x17,0x00,0x00,0x00,0x2c,0x02,0x00,0x00, -0x14,0x00,0x00,0x00,0x2b,0x02,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x2c,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x2e,0x02,0x00,0x00, -0x0f,0x00,0x00,0x00,0x2d,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x32,0x02,0x00,0x00,0x48,0x00,0x00,0x00, -0x2d,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0x0d,0x00,0x00,0x00, -0x34,0x02,0x00,0x00,0x33,0x02,0x00,0x00,0x0c,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x35,0x02,0x00,0x00, -0x34,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x36,0x02,0x00,0x00,0x32,0x02,0x00,0x00,0x35,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x37,0x02,0x00,0x00, -0x2e,0x02,0x00,0x00,0x36,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x39,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x39,0x02,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xa2,0x02,0x00,0x00, -0x3f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0x3c,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x3f,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xbf,0x00,0x00,0x00, -0xf6,0x00,0x04,0x00,0x3b,0x02,0x00,0x00,0x3c,0x02,0x00,0x00, -0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x3f,0x02,0x00,0x00, -0x3a,0x02,0x00,0x00,0x3b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x3a,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x41,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x41,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa3,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x3a,0x02,0x00,0x00,0x9d,0x02,0x00,0x00,0x44,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x47,0x02,0x00,0x00, -0xa3,0x02,0x00,0x00,0x61,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x43,0x02,0x00,0x00,0x44,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x47,0x02,0x00,0x00,0x42,0x02,0x00,0x00, -0x43,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x42,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x02,0x00,0x00, -0xa3,0x02,0x00,0x00,0x62,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4c,0x02,0x00,0x00,0x22,0x02,0x00,0x00, -0x4b,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4e,0x02,0x00,0x00,0x65,0x00,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4f,0x02,0x00,0x00, -0x4c,0x02,0x00,0x00,0x4e,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x53,0x02,0x00,0x00,0xa2,0x02,0x00,0x00, -0xc2,0x01,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x54,0x02,0x00,0x00,0x28,0x02,0x00,0x00,0x53,0x02,0x00,0x00, -0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x56,0x02,0x00,0x00, -0x69,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x57,0x02,0x00,0x00,0x54,0x02,0x00,0x00, -0x56,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0xa5,0x02,0x00,0x00,0x3f,0x00,0x00,0x00, -0x42,0x02,0x00,0x00,0x9b,0x02,0x00,0x00,0x5c,0x02,0x00,0x00, -0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00,0x5f,0x02,0x00,0x00, -0xa5,0x02,0x00,0x00,0xbc,0x00,0x00,0x00,0xf6,0x00,0x04,0x00, -0x5b,0x02,0x00,0x00,0x5c,0x02,0x00,0x00,0x01,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x5f,0x02,0x00,0x00,0x5a,0x02,0x00,0x00, -0x5b,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x5a,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x61,0x02,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xa7,0x02,0x00,0x00,0x3f,0x00,0x00,0x00,0x5a,0x02,0x00,0x00, -0x99,0x02,0x00,0x00,0x64,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x67,0x02,0x00,0x00,0xa7,0x02,0x00,0x00, -0x63,0x00,0x00,0x00,0xf6,0x00,0x04,0x00,0x63,0x02,0x00,0x00, -0x64,0x02,0x00,0x00,0x01,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x67,0x02,0x00,0x00,0x62,0x02,0x00,0x00,0x63,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x62,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x6a,0x02,0x00,0x00,0x4f,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0xb0,0x00,0x05,0x00,0xc2,0x00,0x00,0x00, -0x6d,0x02,0x00,0x00,0x6a,0x02,0x00,0x00,0x37,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x6f,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x6d,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0x6f,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x6e,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x72,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x73,0x02,0x00,0x00,0x14,0x00,0x00,0x00, -0xd0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x74,0x02,0x00,0x00,0x73,0x02,0x00,0x00,0xb0,0x00,0x05,0x00, -0xc2,0x00,0x00,0x00,0x75,0x02,0x00,0x00,0x72,0x02,0x00,0x00, -0x74,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x6f,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x6f,0x02,0x00,0x00,0xf5,0x00,0x07,0x00, -0xc2,0x00,0x00,0x00,0x76,0x02,0x00,0x00,0x6d,0x02,0x00,0x00, -0x62,0x02,0x00,0x00,0x75,0x02,0x00,0x00,0x6e,0x02,0x00,0x00, -0xf7,0x00,0x03,0x00,0x78,0x02,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x76,0x02,0x00,0x00,0x77,0x02,0x00,0x00, -0x78,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x80,0x02,0x00,0x00, -0x57,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x41,0x00,0x05,0x00, -0x17,0x00,0x00,0x00,0x82,0x02,0x00,0x00,0x14,0x00,0x00,0x00, -0x81,0x02,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x83,0x02,0x00,0x00,0x82,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x84,0x02,0x00,0x00,0x80,0x02,0x00,0x00, -0x83,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x85,0x02,0x00,0x00,0x37,0x02,0x00,0x00,0x84,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x87,0x02,0x00,0x00, -0x85,0x02,0x00,0x00,0x4f,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x89,0x02,0x00,0x00,0x87,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x8b,0x02,0x00,0x00,0xa2,0x02,0x00,0x00,0xbc,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8b,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0x84,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x8f,0x02,0x00,0x00,0x8d,0x02,0x00,0x00, -0x8e,0x02,0x00,0x00,0x84,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x91,0x02,0x00,0x00,0xa3,0x02,0x00,0x00,0x63,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x92,0x02,0x00,0x00, -0x8f,0x02,0x00,0x00,0x91,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x94,0x02,0x00,0x00,0x92,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0x41,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0x95,0x02,0x00,0x00,0xca,0x00,0x00,0x00,0x94,0x02,0x00,0x00, -0x3d,0x00,0x04,0x00,0xc4,0x00,0x00,0x00,0x96,0x02,0x00,0x00, -0x95,0x02,0x00,0x00,0x41,0x00,0x06,0x00,0x11,0x01,0x00,0x00, -0x97,0x02,0x00,0x00,0x7c,0x02,0x00,0x00,0x35,0x00,0x00,0x00, -0x89,0x02,0x00,0x00,0x3e,0x00,0x03,0x00,0x97,0x02,0x00,0x00, -0x96,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x78,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x78,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x64,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x64,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x99,0x02,0x00,0x00, -0xa7,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x61,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x63,0x02,0x00,0x00, -0xf9,0x00,0x02,0x00,0x5c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5c,0x02,0x00,0x00,0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x9b,0x02,0x00,0x00,0xa5,0x02,0x00,0x00,0xd0,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x59,0x02,0x00,0x00,0xf8,0x00,0x02,0x00, -0x5b,0x02,0x00,0x00,0xf9,0x00,0x02,0x00,0x44,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x44,0x02,0x00,0x00,0x80,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x9d,0x02,0x00,0x00,0xa3,0x02,0x00,0x00, -0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x41,0x02,0x00,0x00, -0xf8,0x00,0x02,0x00,0x43,0x02,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3c,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3c,0x02,0x00,0x00, -0x80,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x9f,0x02,0x00,0x00, -0xa2,0x02,0x00,0x00,0xd0,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x39,0x02,0x00,0x00,0xf8,0x00,0x02,0x00,0x3b,0x02,0x00,0x00, -0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, -}; -const uint64_t matmul_f32_aligned_fp32_len = 10172; +const uint64_t matmul_f32_f16_fp32_len = 10260; unsigned char matmul_f32_fp32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index d1ba47ac3..aff451b63 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -114,6 +114,7 @@ struct vk_device { size_t idx; vk_matmul_pipeline pipeline_matmul_f32; + vk_matmul_pipeline pipeline_matmul_f32_f16; vk_matmul_pipeline pipeline_matmul_f16; vk_matmul_pipeline pipeline_matmul_f16_f32; vk_pipeline pipeline_matmul_split_k_reduce; @@ -375,13 +376,12 @@ struct ggml_backend_vk_context { vk_context * compute_ctx; vk_context * transfer_ctx; - bool disable; bool initialized; size_t idx; }; -struct vk_instance { +struct vk_instance_t { vk::Instance instance; std::vector device_indices; @@ -423,7 +423,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); static bool vk_instance_initialized = false; -static vk_instance vk_instance; +static vk_instance_t vk_instance; GGML_CALL static void ggml_backend_vk_free(ggml_backend_t backend); @@ -1013,6 +1013,7 @@ static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { uint32_t s_align = 32; ctx->device->pipeline_matmul_f32 = std::make_shared(); + ctx->device->pipeline_matmul_f32_f16 = std::make_shared(); ctx->device->pipeline_matmul_f16_f32 = std::make_shared(); ctx->device->pipeline_matmul_f16 = std::make_shared(); ctx->device->pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0] = std::make_shared(); @@ -1048,6 +1049,13 @@ static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_m, "matmul_f32_aligned_m", matmul_f32_aligned_len, matmul_f32_aligned_data, "main", 3, sizeof(vk_mat_mat_push_constants), m_wg_denoms, warptile_m, m_align); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_s, "matmul_f32_aligned_s", matmul_f32_aligned_len, matmul_f32_aligned_data, "main", 3, sizeof(vk_mat_mat_push_constants), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->l, "matmul_f32_f16_l", matmul_f32_f16_len, matmul_f32_f16_data, "main", 3, sizeof(vk_mat_mat_push_constants), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->m, "matmul_f32_f16_m", matmul_f32_f16_len, matmul_f32_f16_data, "main", 3, sizeof(vk_mat_mat_push_constants), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->s, "matmul_f32_f16_s", matmul_f32_f16_len, matmul_f32_f16_data, "main", 3, sizeof(vk_mat_mat_push_constants), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->a_l, "matmul_f32_f16_aligned_l", matmul_f32_f16_aligned_len, matmul_f32_f16_aligned_data, "main", 3, sizeof(vk_mat_mat_push_constants), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->a_m, "matmul_f32_f16_aligned_m", matmul_f32_f16_aligned_len, matmul_f32_f16_aligned_data, "main", 3, sizeof(vk_mat_mat_push_constants), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->a_s, "matmul_f32_f16_aligned_s", matmul_f32_f16_aligned_len, matmul_f32_f16_aligned_data, "main", 3, sizeof(vk_mat_mat_push_constants), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->l, "matmul_f16_l", matmul_f16_len, matmul_f16_data, "main", 3, sizeof(vk_mat_mat_push_constants), l_wg_denoms, warptile_l, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->m, "matmul_f16_m", matmul_f16_len, matmul_f16_data, "main", 3, sizeof(vk_mat_mat_push_constants), m_wg_denoms, warptile_m, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->s, "matmul_f16_s", matmul_f16_len, matmul_f16_data, "main", 3, sizeof(vk_mat_mat_push_constants), s_wg_denoms, warptile_s, 1); @@ -1230,6 +1238,13 @@ static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_m, "matmul_f32_aligned_m", matmul_f32_aligned_fp32_len, matmul_f32_aligned_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), m_wg_denoms, warptile_m, m_align); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32->a_s, "matmul_f32_aligned_s", matmul_f32_aligned_fp32_len, matmul_f32_aligned_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->l, "matmul_f32_f16_l", matmul_f32_f16_fp32_len, matmul_f32_f16_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), l_wg_denoms, warptile_l, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->m, "matmul_f32_f16_m", matmul_f32_f16_fp32_len, matmul_f32_f16_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), m_wg_denoms, warptile_m, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->s, "matmul_f32_f16_s", matmul_f32_f16_fp32_len, matmul_f32_f16_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), s_wg_denoms, warptile_s, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->a_l, "matmul_f32_f16_aligned_l", matmul_f32_f16_aligned_fp32_len, matmul_f32_f16_aligned_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), l_wg_denoms, warptile_l, l_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->a_m, "matmul_f32_f16_aligned_m", matmul_f32_f16_aligned_fp32_len, matmul_f32_f16_aligned_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), m_wg_denoms, warptile_m, m_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f32_f16->a_s, "matmul_f32_f16_aligned_s", matmul_f32_f16_aligned_fp32_len, matmul_f32_f16_aligned_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), s_wg_denoms, warptile_s, s_align); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->l, "matmul_f16_l", matmul_f16_fp32_len, matmul_f16_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), l_wg_denoms, warptile_l, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->m, "matmul_f16_m", matmul_f16_fp32_len, matmul_f16_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), m_wg_denoms, warptile_m, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_matmul_f16->s, "matmul_f16_s", matmul_f16_fp32_len, matmul_f16_fp32_data, "main", 3, sizeof(vk_mat_mat_push_constants), s_wg_denoms, warptile_s, 1); @@ -1859,7 +1874,6 @@ static void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { ctx->compute_ctx = nullptr; ctx->transfer_ctx = nullptr; - ctx->disable = false; ctx->initialized = true; ctx->idx = idx; @@ -1903,6 +1917,9 @@ static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_pipeline(ggml_backend_vk_conte if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F32) { return ctx->device->pipeline_matmul_f32; } + if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_F16) { + return ctx->device->pipeline_matmul_f32_f16; + } if (src0_type == GGML_TYPE_F16 && src1_type == GGML_TYPE_F32) { return ctx->device->pipeline_matmul_f16_f32; } @@ -2722,7 +2739,7 @@ static void ggml_vk_matmul( uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t expert_stride_b, uint32_t expert_stride_d, uint32_t idx, uint32_t nbi1, uint32_t n_as) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), c: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << split_k_buffer.buffer->buffer << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ")" << std::endl; + std::cerr << "ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), c: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ")" << std::endl; #endif ggml_vk_sync_buffers(subctx); if (split_k == 1) { @@ -2792,7 +2809,7 @@ static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, ggml_ static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context * subctx, vk_pipeline pipeline, const ggml_tensor * tensor, vk_subbuffer&& in, vk_subbuffer&& out) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", backend=" << tensor->backend << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; + std::cerr << "ggml_vk_cpy_to_contiguous((" << tensor << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << "), "; std::cerr << "buffer in size=" << in.buffer->size << ", buffer out size=" << out.buffer->size << ")" << std::endl; #endif const int tensor_type_size = ggml_type_size(tensor->type); @@ -2812,9 +2829,9 @@ static void ggml_vk_cpy_to_contiguous(ggml_backend_vk_context * ctx, vk_context static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; + std::cerr << "ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; #endif GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); // NOLINT GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT @@ -2982,19 +2999,13 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context * su ne01, ne11, ne10, ne10, ne10, ne01, split_k, ne12*ne13, ne02, ne12, r2, r3, stride_batch_x, stride_batch_y, ne20*ne21, 0, 0, 0, 0, 1 ); // NOLINT - - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - // copy dst to host - float * d = (float *) ((char *) dst->data); - ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, d, sizeof(float) * d_ne * ne12 * ne13); - } } static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; + std::cerr << "ggml_vk_mul_mat_vec_q_f16((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; #endif GGML_ASSERT(ggml_vk_dim01_contiguous(src0) || src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); // NOLINT GGML_ASSERT(ggml_vk_dim01_contiguous(src1) || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16); // NOLINT @@ -3147,12 +3158,11 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_mul_mat_p021_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; + std::cerr << "ggml_vk_mul_mat_p021_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; #endif GGML_ASSERT(ggml_is_permuted(src0) && ggml_is_permuted(src1)); - GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); GGML_ASSERT(src0->nb[0] <= src0->nb[1] && src0->nb[2] <= src0->nb[3]); // NOLINT GGML_ASSERT(src1->nb[0] <= src1->nb[1] && src1->nb[2] <= src1->nb[3]); // NOLINT GGML_ASSERT(src0->type == GGML_TYPE_F16); @@ -3217,25 +3227,17 @@ static void ggml_vk_mul_mat_vec_p021_f16_f32(ggml_backend_vk_context * ctx, vk_c const std::array pc = { (uint32_t)ne00, (uint32_t)ne01, (uint32_t)ne02, (uint32_t)ne12, (uint32_t)(qy_shader_offset / ggml_type_size(src1->type)), (uint32_t)(d_shader_offset / ggml_type_size(dst->type)) }; ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_mul_mat_vec_p021_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 6 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); - - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - // copy dst to host - float * d = (float *) dst->data; - ggml_vk_sync_buffers(subctx); - ggml_vk_buffer_read_async(ctx, subctx, d_D, d_buf_offset, d, sizeof(float) * d_ne); - } } static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_mul_mat_nc_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; + std::cerr << "ggml_vk_mul_mat_nc_f16_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "),)" << std::endl; #endif GGML_ASSERT(!ggml_is_transposed(src0)); GGML_ASSERT(!ggml_is_transposed(src1)); GGML_ASSERT(!ggml_is_permuted(src0)); - GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); GGML_ASSERT(src0->type == GGML_TYPE_F16); GGML_ASSERT(src1->type == GGML_TYPE_F32); @@ -3302,26 +3304,6 @@ static void ggml_vk_mul_mat_vec_nc_f16_f32(ggml_backend_vk_context * ctx, vk_con const std::array pc = { (uint32_t)ne00, (uint32_t)ne01, row_stride_x, channel_stride_x, (uint32_t)(ne12 / ne02), (uint32_t)(qy_shader_offset / ggml_type_size(src1->type)), (uint32_t)(d_shader_offset / ggml_type_size(dst->type)) }; ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_mul_mat_vec_nc_f16_f32, { { d_Qx, qx_buf_offset, qx_sz }, { d_Qy, qy_buffer_offset, qy_sz + qy_shader_offset }, { d_D, d_buffer_offset, d_sz + d_shader_offset } }, 7 * sizeof(uint32_t), &pc, { 1, (uint32_t)ne01, (uint32_t)ne12 }); - - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - // copy dst to host - float * d = (float *) dst->data; - ggml_vk_sync_buffers(subctx); - ggml_vk_buffer_read_async(ctx, subctx, d_D, d_buf_offset, d, sizeof(float) * d_ne); - } -} - -static bool ggml_vk_can_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { - const uint64_t ne10 = src1->ne[0]; - - const uint64_t ne0 = dst->ne[0]; - const uint64_t ne1 = dst->ne[1]; - - // TODO: find the optimal values for these - return (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && - (src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || ggml_is_quantized(src1->type)) && - dst->type == GGML_TYPE_F32 && - ((ne0 >= 32 && ne1 >= 32 && ne10 >= 32) || src0->backend == GGML_BACKEND_TYPE_GPU); } static void ggml_vk_mul_mat(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -3711,8 +3693,6 @@ static void ggml_vk_op_repeat(ggml_backend_vk_context * ctx, vk_context * subctx // TODO: support for transposed / permuted tensors GGML_ASSERT(nb0 == sizeof(float)); GGML_ASSERT(nb00 == sizeof(float)); - GGML_ASSERT(src0->backend == GGML_BACKEND_TYPE_GPU); - GGML_ASSERT(dst->backend == GGML_BACKEND_TYPE_GPU); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; @@ -3902,11 +3882,11 @@ static bool ggml_vk_op_supports_incontiguous(ggml_op op) { template static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op, const PC&& pc) { #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", backend=" << src0->backend << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; + std::cerr << "ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; if (src1 != nullptr) { - std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", backend=" << src1->backend << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; + std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; } - std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", backend=" << dst->backend << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "), " << ggml_op_name(op) << ")" << std::endl; + std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "), " << ggml_op_name(op) << ")" << std::endl; #endif GGML_ASSERT(op == GGML_OP_GET_ROWS || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT GGML_ASSERT(op == GGML_OP_CPY || ggml_vk_dim01_contiguous(src0)); // NOLINT @@ -3923,8 +3903,6 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c const uint64_t ne13 = use_src1 ? src1->ne[3] : 0; const uint64_t ne1 = ne10 * ne11; // const uint64_t nb10 = use_src1 ? src1->nb[0] : 0; - const uint64_t nb2 = dst->nb[2]; - const uint64_t nb3 = dst->nb[3]; vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, dst, op); ggml_vk_func_t op_func; @@ -3975,7 +3953,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c vk_buffer d_D = extra->buffer_gpu.lock(); // Workaround for tiny tensor inputs on ROPE - if (use_src1 && src1->backend == GGML_BACKEND_TYPE_GPU && y_sz > d_D->size) { + if (use_src1 && y_sz > d_D->size) { y_sz = VK_WHOLE_SIZE; } @@ -4066,13 +4044,6 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } - if (dst->backend == GGML_BACKEND_TYPE_CPU && op == GGML_OP_CPY) { - ggml_vk_d2h_tensor_2d(ctx, subctx, d_D, 0, dst); - } else if(dst->backend == GGML_BACKEND_TYPE_CPU) { - // copy dst to host - float * d = (float *) dst->data; - ggml_vk_buffer_read_async(ctx, subctx, d_D, 0, d, d_sz); - } } else { GGML_ASSERT(op != GGML_OP_SOFT_MAX); GGML_ASSERT(op != GGML_OP_ARGSORT); @@ -4111,10 +4082,6 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset + x_offset, x_sz }, { d_D, d_buf_offset + d_offset, d_sz } }, sizeof(PC), &pc, elements); } - if (dst->backend == GGML_BACKEND_TYPE_CPU) { - // copy dst to host - ggml_vk_buffer_read_async(ctx, subctx, d_D, d_buf_offset + d_offset, (char *) dst->data + i02*nb2 + i03*nb3, d_sz); - } } } } @@ -4377,6 +4344,9 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f32->a_s; shname = "F32_ALIGNED_S"; + } else if (std::is_same() && std::is_same()) { + p = ctx->device->pipeline_matmul_f32_f16->a_s; + shname = "F32_F16_ALIGNED_S"; } else if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f16_f32->a_s; shname = "F16_F32_ALIGNED_S"; @@ -4390,6 +4360,9 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f32->a_m; shname = "F32_ALIGNED_M"; + } else if (std::is_same() && std::is_same()) { + p = ctx->device->pipeline_matmul_f32_f16->a_m; + shname = "F32_F16_ALIGNED_M"; } else if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f16_f32->a_m; shname = "F16_F32_ALIGNED_M"; @@ -4403,6 +4376,9 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f32->a_l; shname = "F32_ALIGNED_L"; + } else if (std::is_same() && std::is_same()) { + p = ctx->device->pipeline_matmul_f32_f16->a_l; + shname = "F32_F16_ALIGNED_L"; } else if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f16_f32->a_l; shname = "F16_F32_ALIGNED_L"; @@ -4423,6 +4399,9 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f32->s; shname = "F32_S"; + } else if (std::is_same() && std::is_same()) { + p = ctx->device->pipeline_matmul_f32_f16->s; + shname = "F32_F16_S"; } else if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f16_f32->s; shname = "F16_F32_S"; @@ -4434,6 +4413,9 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f32->m; shname = "F32_M"; + } else if (std::is_same() && std::is_same()) { + p = ctx->device->pipeline_matmul_f32_f16->m; + shname = "F32_F16_M"; } else if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f16_f32->m; shname = "F16_F32_M"; @@ -4445,6 +4427,9 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f32->l; shname = "F32_L"; + } else if (std::is_same() && std::is_same()) { + p = ctx->device->pipeline_matmul_f32_f16->l; + shname = "F32_F16_L"; } else if (std::is_same() && std::is_same()) { p = ctx->device->pipeline_matmul_f16_f32->l; shname = "F16_F32_L"; @@ -4557,15 +4542,11 @@ static void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t src1_ggml->data = y; tensor_ggml->data = d_chk; - ctx->disable = true; - ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); ggml_build_forward_expand(cgraph, tensor_ggml); ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1); - ctx->disable = false; - ggml_free(ggml_ctx); double avg_err = 0.0; @@ -5045,15 +5026,11 @@ static void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, src1_ggml->data = y; tensor_ggml->data = d_chk; - ctx->disable = true; - ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); ggml_build_forward_expand(cgraph, tensor_ggml); ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 1); - ctx->disable = false; - ggml_free(ggml_ctx); double avg_err = 0.0; @@ -5130,12 +5107,12 @@ static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggm #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_preallocate_buffers_graph(" << node << ")" << std::endl; #endif - if (ctx->disable || node->backend != GGML_BACKEND_TYPE_GPU) { + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) node->extra; + + if (extra == nullptr) { return; } - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) node->extra; - ggml_tensor * src0 = node->src[0]; ggml_tensor * src1 = node->src[1]; @@ -5240,9 +5217,6 @@ static void ggml_vk_preallocate_buffers_graph(ggml_backend_vk_context * ctx, ggm } static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) { - if (ctx->disable) { - return; - } #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_preallocate_buffers(x_size: " << ctx->prealloc_size_x << " y_size: " << ctx->prealloc_size_y << " split_k_size: " << ctx->prealloc_size_split_k << ")" << std::endl; #endif @@ -5416,7 +5390,9 @@ static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx) { } static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * node, bool last_node){ - if (ctx->disable || node->backend != GGML_BACKEND_TYPE_GPU || ggml_is_empty(node)) { + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) node->extra; + + if (ggml_is_empty(node) || extra == nullptr) { return; } @@ -5429,8 +5405,6 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod const ggml_tensor * src0 = node->src[0]; const ggml_tensor * src1 = node->src[1]; - ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) node->extra; - switch (node->op) { case GGML_OP_UNARY: switch (ggml_get_unary_op(node)) { @@ -5575,7 +5549,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod last_node = true; #endif - if (node->backend == GGML_BACKEND_TYPE_CPU || last_node) { + if (last_node) { ggml_vk_ctx_end(ctx->compute_ctx); ctx->compute_ctx->exit_tensor = node; ctx->compute_ctx = nullptr; @@ -5583,10 +5557,6 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod } static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_params * params, ggml_tensor * tensor){ - if (ctx->disable) { - return false; - } - ggml_tensor_extra_gpu * extra = nullptr; switch (tensor->op) { @@ -5645,7 +5615,7 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_ } #ifdef GGML_VULKAN_DEBUG - std::cerr << "ggml_vk_compute_forward(" << tensor << ", name=" << tensor->name << ", op=" << ggml_op_name(tensor->op) << ", type=" << tensor->type << ", backend=" << tensor->backend << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << ", view_src=" << tensor->view_src << ", view_offs=" << tensor->view_offs << ")" << std::endl; + std::cerr << "ggml_vk_compute_forward(" << tensor << ", name=" << tensor->name << ", op=" << ggml_op_name(tensor->op) << ", type=" << tensor->type << ", ne0=" << tensor->ne[0] << ", ne1=" << tensor->ne[1] << ", ne2=" << tensor->ne[2] << ", ne3=" << tensor->ne[3] << ", nb0=" << tensor->nb[0] << ", nb1=" << tensor->nb[1] << ", nb2=" << tensor->nb[2] << ", nb3=" << tensor->nb[3] << ", view_src=" << tensor->view_src << ", view_offs=" << tensor->view_offs << ")" << std::endl; #endif #ifdef GGML_VULKAN_CHECK_RESULTS @@ -5685,9 +5655,6 @@ static bool ggml_vk_compute_forward(ggml_backend_vk_context * ctx, ggml_compute_ // Clean up after graph processing is done static void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { - if (ctx->disable) { - return; - } #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_graph_cleanup()" << std::endl; #endif @@ -5860,7 +5827,6 @@ GGML_CALL static void ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t b extra->offset = (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base; } - tensor->backend = GGML_BACKEND_TYPE_GPU; tensor->extra = extra; } @@ -5868,8 +5834,6 @@ GGML_CALL static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t bu #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")" << std::endl; #endif - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; @@ -5883,8 +5847,6 @@ GGML_CALL static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t bu #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")" << std::endl; #endif - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); - ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context; ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; @@ -6027,6 +5989,7 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_bu #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")" << std::endl; #endif + size += 32; // Behave like the CPU buffer type void * ptr = nullptr; try { ptr = ggml_vk_host_malloc(&vk_instance.contexts[0], size); @@ -6114,7 +6077,6 @@ GGML_CALL static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, g #endif ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type(ctx->idx) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; @@ -6135,7 +6097,6 @@ GGML_CALL static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, c #endif ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_buffer_type(ctx->idx) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type"); - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_GPU); ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra; @@ -6201,6 +6162,10 @@ GGML_CALL static void ggml_backend_vk_synchronize(ggml_backend_t backend) { ctx->transfer_ctx = nullptr; } +static bool ggml_vk_is_empty(ggml_tensor * node) { + return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE; +} + GGML_CALL static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)" << std::endl; @@ -6215,7 +6180,7 @@ GGML_CALL static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backen int last_node = cgraph->n_nodes - 1; // If the last op in the cgraph isn't backend GPU, the command buffer doesn't get closed properly - while (last_node > 0 && (cgraph->nodes[last_node]->backend != GGML_BACKEND_TYPE_GPU || ggml_is_empty(cgraph->nodes[last_node]))) { + while (last_node > 0 && ggml_vk_is_empty(cgraph->nodes[last_node])) { last_node -= 1; } @@ -6229,7 +6194,7 @@ GGML_CALL static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backen for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; - if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { + if (ggml_vk_is_empty(node)) { continue; } @@ -6873,16 +6838,11 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ GGML_ASSERT(false); } - // Disable vulkan here to avoid the hooks in ggml.c - ctx->disable = true; - ggml_cgraph * cgraph = ggml_new_graph(ggml_ctx); ggml_build_forward_expand(cgraph, tensor_clone); ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 8); - ctx->disable = false; - ggml_vk_check_tensor(ggml_op_name(tensor->op), tensor_clone); if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { ggml_vk_print_tensor(ctx, tensor_clone, "tensor_clone"); diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index 3c22a986c..8096c03b7 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -2877,13 +2877,16 @@ async def main(): stream.clear() stream.extend((mulmat_head, shader_float_type, mulmat_body1, mulmat_load_scalar, mulmat_body2)) tasks.append(string_to_spv("matmul_f32", "".join(stream), {"A_TYPE": "float", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f32_aligned", "".join(stream), {"LOAD_VEC_A": 1, "LOAD_VEC_B": load_vec, "A_TYPE": "float", "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f32_aligned", "".join(stream), {"LOAD_VEC_A": load_vec, "LOAD_VEC_B": load_vec, "A_TYPE": vec_type, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + + tasks.append(string_to_spv("matmul_f32_f16", "".join(stream), {"A_TYPE": "float", "B_TYPE": "float16_t", "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f32_f16_aligned", "".join(stream), {"LOAD_VEC_A": load_vec, "LOAD_VEC_B": load_vec, "A_TYPE": vec_type, "B_TYPE": vec_type_f16, "D_TYPE": "float"}, fp16)) tasks.append(string_to_spv("matmul_f16", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float16_t", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_aligned", "".join(stream), {"LOAD_VEC_A": 1, "LOAD_VEC_B": load_vec, "A_TYPE": "float16_t", "B_TYPE": vec_type_f16, "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f16_aligned", "".join(stream), {"LOAD_VEC_A": load_vec, "LOAD_VEC_B": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type_f16, "D_TYPE": "float"}, fp16)) tasks.append(string_to_spv("matmul_f16_f32", "".join(stream), {"A_TYPE": "float16_t", "B_TYPE": "float", "D_TYPE": "float"}, fp16)) - tasks.append(string_to_spv("matmul_f16_f32_aligned", "".join(stream), {"LOAD_VEC_A": 1, "LOAD_VEC_B": load_vec, "A_TYPE": "float16_t", "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) + tasks.append(string_to_spv("matmul_f16_f32_aligned", "".join(stream), {"LOAD_VEC_A": load_vec, "LOAD_VEC_B": load_vec, "A_TYPE": vec_type_f16, "B_TYPE": vec_type, "D_TYPE": "float"}, fp16)) stream.clear() stream.extend((mulmat_head, shader_int8_ext, shader_float_type, shader_q4_0_defines, mulmat_body1, mulmat_load_q4_0, mulmat_body2)) From 1ea2a0036e88172d6c8bf7e1a1989a03894dc955 Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Sun, 19 May 2024 11:37:04 -0500 Subject: [PATCH 078/137] quantize : fix --keep-split check (#7374) --- examples/quantize/quantize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 909eab283..cbb452334 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -284,7 +284,7 @@ int main(int argc, char ** argv) { } else { usage(argv[0]); } - } else if (strcmp(argv[arg_idx], "--keep-split")) { + } else if (strcmp(argv[arg_idx], "--keep-split") == 0) { params.keep_split = true; } else { usage(argv[0]); From d359f30921a9f62a0fd299c412ff3f270286fea6 Mon Sep 17 00:00:00 2001 From: slaren Date: Mon, 20 May 2024 01:17:03 +0200 Subject: [PATCH 079/137] llama : remove MPI backend (#7395) --- .devops/nix/package.nix | 1 - .github/workflows/build.yml | 34 ------ CMakeLists.txt | 33 +----- Makefile | 12 -- README.md | 39 ------- ggml-mpi.c | 216 ----------------------------------- ggml-mpi.h | 39 ------- llama.cpp | 48 +------- scripts/LlamaConfig.cmake.in | 5 - 9 files changed, 2 insertions(+), 425 deletions(-) delete mode 100644 ggml-mpi.c delete mode 100644 ggml-mpi.h diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 1c9633cdf..e8d5b0bd9 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -214,7 +214,6 @@ effectiveStdenv.mkDerivation ( (cmakeBool "LLAMA_CUDA" useCuda) (cmakeBool "LLAMA_HIPBLAS" useRocm) (cmakeBool "LLAMA_METAL" useMetalKit) - (cmakeBool "LLAMA_MPI" useMpi) (cmakeBool "LLAMA_VULKAN" useVulkan) (cmakeBool "LLAMA_STATIC" enableStatic) ] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53e61b80f..7b616281b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -306,40 +306,6 @@ jobs: cd build ctest -L main --verbose --timeout 900 - ubuntu-latest-cmake-mpi: - runs-on: ubuntu-latest - - continue-on-error: true - - strategy: - matrix: - mpi_library: [mpich, libopenmpi-dev] - - steps: - - name: Clone - id: checkout - uses: actions/checkout@v4 - - - name: Dependencies - id: depends - run: | - sudo apt-get update - sudo apt-get install build-essential ${{ matrix.mpi_library }} - - - name: Build - id: cmake_build - run: | - mkdir build - cd build - cmake -DLLAMA_MPI=ON .. - cmake --build . --config Release -j $(nproc) - - - name: Test - id: cmake_test - run: | - cd build - ctest -L main --verbose - ubuntu-latest-cmake-rpc: runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index cbeb2ee37..616698c7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,7 +122,6 @@ set(LLAMA_METAL_MACOSX_VERSION_MIN "" CACHE STRING "llama: metal minimum macOS version") set(LLAMA_METAL_STD "" CACHE STRING "llama: metal standard version (-std flag)") option(LLAMA_KOMPUTE "llama: use Kompute" OFF) -option(LLAMA_MPI "llama: use MPI" OFF) option(LLAMA_RPC "llama: use RPC" OFF) option(LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF) option(LLAMA_SYCL "llama: use SYCL" OFF) @@ -466,35 +465,6 @@ if (LLAMA_CUDA) endif() endif() -if (LLAMA_MPI) - cmake_minimum_required(VERSION 3.10) - find_package(MPI) - if (MPI_C_FOUND) - message(STATUS "MPI found") - - set(GGML_HEADERS_MPI ggml-mpi.h) - set(GGML_SOURCES_MPI ggml-mpi.c) - - add_compile_definitions(GGML_USE_MPI) - add_compile_definitions(${MPI_C_COMPILE_DEFINITIONS}) - - if (NOT MSVC) - add_compile_options(-Wno-cast-qual) - endif() - - set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${MPI_C_LIBRARIES}) - set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${MPI_C_INCLUDE_DIRS}) - - # Even if you're only using the C header, C++ programs may bring in MPI - # C++ functions, so more linkage is needed - if (MPI_CXX_FOUND) - set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} ${MPI_CXX_LIBRARIES}) - endif() - else() - message(WARNING "MPI not found") - endif() -endif() - if (LLAMA_RPC) add_compile_definitions(GGML_USE_RPC) @@ -1218,7 +1188,6 @@ add_library(ggml OBJECT ${GGML_SOURCES_CUDA} ${GGML_HEADERS_CUDA} ${GGML_SOURCES_OPENCL} ${GGML_HEADERS_OPENCL} ${GGML_SOURCES_METAL} ${GGML_HEADERS_METAL} - ${GGML_SOURCES_MPI} ${GGML_HEADERS_MPI} ${GGML_SOURCES_RPC} ${GGML_HEADERS_RPC} ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA} ${GGML_SOURCES_SYCL} ${GGML_HEADERS_SYCL} @@ -1306,7 +1275,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/LlamaConfig.cmake set(GGML_PUBLIC_HEADERS "ggml.h" "ggml-alloc.h" "ggml-backend.h" "${GGML_HEADERS_CUDA}" "${GGML_HEADERS_OPENCL}" - "${GGML_HEADERS_METAL}" "${GGML_HEADERS_MPI}" "${GGML_HEADERS_EXTRA}") + "${GGML_HEADERS_METAL}" "${GGML_HEADERS_EXTRA}") set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}") install(TARGETS ggml PUBLIC_HEADER) diff --git a/Makefile b/Makefile index 22d521856..9a26aec50 100644 --- a/Makefile +++ b/Makefile @@ -399,13 +399,6 @@ ifndef LLAMA_NO_ACCELERATE endif endif # LLAMA_NO_ACCELERATE -ifdef LLAMA_MPI - MK_CPPFLAGS += -DGGML_USE_MPI - MK_CFLAGS += -Wno-cast-qual - MK_CXXFLAGS += -Wno-cast-qual - OBJS += ggml-mpi.o -endif # LLAMA_MPI - ifdef LLAMA_OPENBLAS MK_CPPFLAGS += -DGGML_USE_OPENBLAS $(shell pkg-config --cflags-only-I openblas) MK_CFLAGS += $(shell pkg-config --cflags-only-other openblas) @@ -629,11 +622,6 @@ ggml-metal-embed.o: ggml-metal.metal ggml-common.h endif endif # LLAMA_METAL -ifdef LLAMA_MPI -ggml-mpi.o: ggml-mpi.c ggml-mpi.h - $(CC) $(CFLAGS) -c $< -o $@ -endif # LLAMA_MPI - ifndef LLAMA_NO_LLAMAFILE sgemm.o: sgemm.cpp sgemm.h ggml.h $(CXX) $(CXXFLAGS) -c $< -o $@ diff --git a/README.md b/README.md index 7dd6fc0eb..4abfd6d7e 100644 --- a/README.md +++ b/README.md @@ -382,45 +382,6 @@ To disable the Metal build at compile time use the `LLAMA_NO_METAL=1` flag or th When built with Metal support, you can explicitly disable GPU inference with the `--n-gpu-layers|-ngl 0` command-line argument. -### MPI Build - -MPI lets you distribute the computation over a cluster of machines. Because of the serial nature of LLM prediction, this won't yield any end-to-end speed-ups, but it will let you run larger models than would otherwise fit into RAM on a single machine. - -First you will need MPI libraries installed on your system. The two most popular (only?) options are [MPICH](https://www.mpich.org) and [OpenMPI](https://www.open-mpi.org). Either can be installed with a package manager (`apt`, Homebrew, MacPorts, etc). - -Next you will need to build the project with `LLAMA_MPI` set to true on all machines; if you're building with `make`, you will also need to specify an MPI-capable compiler (when building with CMake, this is configured automatically): - -- Using `make`: - - ```bash - make CC=mpicc CXX=mpicxx LLAMA_MPI=1 - ``` - -- Using `CMake`: - - ```bash - cmake -S . -B build -DLLAMA_MPI=ON - ``` - -Once the programs are built, download/convert the weights on all of the machines in your cluster. The paths to the weights and programs should be identical on all machines. - -Next, ensure password-less SSH access to each machine from the primary host, and create a `hostfile` with a list of the hostnames and their relative "weights" (slots). If you want to use localhost for computation, use its local subnet IP address rather than the loopback address or "localhost". - -Here is an example hostfile: - -``` -192.168.0.1:2 -malvolio.local:1 -``` - -The above will distribute the computation across 2 processes on the first host and 1 process on the second host. Each process will use roughly an equal amount of RAM. Try to keep these numbers small, as inter-process (intra-host) communication is expensive. - -Finally, you're ready to run a computation using `mpirun`: - -```bash -mpirun -hostfile hostfile -n 3 ./main -m ./models/7B/ggml-model-q4_0.gguf -n 128 -``` - ### BLAS Build Building the program with BLAS support may lead to some performance improvements in prompt processing using batch sizes higher than 32 (the default is 512). Support with CPU-only BLAS implementations doesn't affect the normal generation performance. We may see generation performance improvements with GPU-involved BLAS implementations, e.g. cuBLAS, hipBLAS and CLBlast. There are currently several different BLAS implementations available for build and use: diff --git a/ggml-mpi.c b/ggml-mpi.c deleted file mode 100644 index ae176d707..000000000 --- a/ggml-mpi.c +++ /dev/null @@ -1,216 +0,0 @@ -#include "ggml-mpi.h" - -#include "ggml.h" - -#include - -#include -#include - -#define MIN(a, b) ((a) < (b) ? (a) : (b)) - -#define UNUSED GGML_UNUSED - -struct ggml_mpi_context { - int rank; - int size; -}; - -void ggml_mpi_backend_init(void) { - MPI_Init(NULL, NULL); -} - -void ggml_mpi_backend_free(void) { - MPI_Finalize(); -} - -struct ggml_mpi_context * ggml_mpi_init(void) { - struct ggml_mpi_context * ctx = calloc(1, sizeof(struct ggml_mpi_context)); - - MPI_Comm_rank(MPI_COMM_WORLD, &ctx->rank); - MPI_Comm_size(MPI_COMM_WORLD, &ctx->size); - - return ctx; -} - -void ggml_mpi_free(struct ggml_mpi_context * ctx) { - free(ctx); -} - -int ggml_mpi_rank(struct ggml_mpi_context * ctx) { - return ctx->rank; -} - -void ggml_mpi_eval_init( - struct ggml_mpi_context * ctx_mpi, - int * n_tokens, - int * n_past, - int * n_threads) { - UNUSED(ctx_mpi); - - // synchronize the worker node parameters with the root node - MPI_Barrier(MPI_COMM_WORLD); - - MPI_Bcast(n_tokens, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(n_past, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(n_threads, 1, MPI_INT, 0, MPI_COMM_WORLD); -} - -static int ggml_graph_get_node_idx(struct ggml_cgraph * gf, const char * name) { - struct ggml_tensor * t = ggml_graph_get_tensor(gf, name); - if (t == NULL) { - fprintf(stderr, "%s: tensor %s not found\n", __func__, name); - return -1; - } - - for (int i = 0; i < gf->n_nodes; i++) { - if (gf->nodes[i] == t) { - return i; - } - } - - fprintf(stderr, "%s: tensor %s not found in graph (should not happen)\n", __func__, name); - return -1; -} - -static void ggml_mpi_tensor_send(struct ggml_tensor * t, int mpi_rank_dst) { - MPI_Datatype mpi_type; - - switch (t->type) { - case GGML_TYPE_I32: mpi_type = MPI_INT32_T; break; - case GGML_TYPE_F32: mpi_type = MPI_FLOAT; break; - default: GGML_ASSERT(false && "not implemented"); - } - - const int retval = MPI_Send(t->data, ggml_nelements(t), mpi_type, mpi_rank_dst, 0, MPI_COMM_WORLD); - GGML_ASSERT(retval == MPI_SUCCESS); -} - -static void ggml_mpi_tensor_recv(struct ggml_tensor * t, int mpi_rank_src) { - MPI_Datatype mpi_type; - - switch (t->type) { - case GGML_TYPE_I32: mpi_type = MPI_INT32_T; break; - case GGML_TYPE_F32: mpi_type = MPI_FLOAT; break; - default: GGML_ASSERT(false && "not implemented"); - } - - MPI_Status status; UNUSED(status); - - const int retval = MPI_Recv(t->data, ggml_nelements(t), mpi_type, mpi_rank_src, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - GGML_ASSERT(retval == MPI_SUCCESS); -} - -// TODO: there are many improvements that can be done to this implementation -void ggml_mpi_graph_compute_pre( - struct ggml_mpi_context * ctx_mpi, - struct ggml_cgraph * gf, - int n_layers) { - const int mpi_rank = ctx_mpi->rank; - const int mpi_size = ctx_mpi->size; - - struct ggml_tensor * inp_tokens = ggml_graph_get_tensor(gf, "inp_tokens"); - if (inp_tokens == NULL) { - fprintf(stderr, "%s: tensor 'inp_tokens' not found\n", __func__); - return; - } - - struct ggml_tensor * inp0 = ggml_graph_get_tensor(gf, "layer_inp_0"); - if (inp0 == NULL) { - fprintf(stderr, "%s: tensor 'inp0' not found\n", __func__); - return; - } - - GGML_ASSERT(inp0 == gf->nodes[0]); - - // distribute the compute graph into slices across the MPI nodes - // - // the main node (0) processes the last layers + the remainder of the compute graph - // and is responsible to pass the input tokens to the first node (1) - // - // node 1: [( 0) * n_per_node, ( 1) * n_per_node) - // node 2: [( 1) * n_per_node, ( 2) * n_per_node) - // ... - // node n-1: [(n-2) * n_per_node, (n-1) * n_per_node) - // node 0: [(n-1) * n_per_node, n_nodes) - // - if (mpi_rank > 0) { - if (mpi_rank == 1) { - // the first node (1) receives the input tokens from the main node (0) - ggml_mpi_tensor_recv(inp_tokens, 0); - } else { - // recv input data for each node into the "inp0" tensor (i.e. the first node in the compute graph) - ggml_mpi_tensor_recv(inp0, mpi_rank - 1); - } - } else if (mpi_size > 1) { - // node 0 sends the input tokens to node 1 - ggml_mpi_tensor_send(inp_tokens, 1); - - // recv the output data from the last node - ggml_mpi_tensor_recv(inp0, mpi_size - 1); - } - - { - const int n_per_node = (n_layers + (mpi_size - 1)) / mpi_size; - - const int mpi_idx = mpi_rank > 0 ? mpi_rank - 1 : mpi_size - 1; - - const int il0 = (mpi_idx + 0) * n_per_node; - const int il1 = MIN(n_layers, (mpi_idx + 1) * n_per_node); - - char name_l0[GGML_MAX_NAME]; - char name_l1[GGML_MAX_NAME]; - - snprintf(name_l0, sizeof(name_l0), "layer_inp_%d", il0); - snprintf(name_l1, sizeof(name_l1), "layer_inp_%d", il1); - - const int idx_l0 = ggml_graph_get_node_idx(gf, name_l0); - const int idx_l1 = mpi_rank > 0 ? ggml_graph_get_node_idx(gf, name_l1) + 1 : gf->n_nodes; - - if (idx_l0 < 0 || idx_l1 < 0) { - fprintf(stderr, "%s: layer input nodes not found\n", __func__); - return; - } - - // attach the input data to all nodes that need it - // TODO: not great - should be able to do this without modifying the compute graph (see next TODO below) - for (int i = idx_l0; i < idx_l1; i++) { - if (gf->nodes[i]->src[0] == gf->nodes[idx_l0]) { - gf->nodes[i]->src[0] = inp0; - } - if (gf->nodes[i]->src[1] == gf->nodes[idx_l0]) { - gf->nodes[i]->src[1] = inp0; - } - } - - // TODO: instead of rearranging the nodes, we should be able to execute a subset of the compute graph - for (int i = 1; i < idx_l1 - idx_l0; i++) { - gf->nodes[i] = gf->nodes[idx_l0 + i]; - gf->grads[i] = gf->grads[idx_l0 + i]; - } - - // the first node performs the "get_rows" operation, the rest of the nodes get the data from the previous node - if (mpi_idx != 0) { - gf->nodes[0]->op = GGML_OP_NONE; - } - - gf->n_nodes = idx_l1 - idx_l0; - - //fprintf(stderr, "%s: node %d: processing %d nodes [%d, %d)\n", __func__, mpi_rank, gf->n_nodes, il0, il1); - } -} - -void ggml_mpi_graph_compute_post( - struct ggml_mpi_context * ctx_mpi, - struct ggml_cgraph * gf, - int n_layers) { - UNUSED(n_layers); - - const int mpi_rank = ctx_mpi->rank; - const int mpi_size = ctx_mpi->size; - - // send the output data to the next node - if (mpi_rank > 0) { - ggml_mpi_tensor_send(gf->nodes[gf->n_nodes - 1], (mpi_rank + 1) % mpi_size); - } -} diff --git a/ggml-mpi.h b/ggml-mpi.h deleted file mode 100644 index eda119d44..000000000 --- a/ggml-mpi.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -struct ggml_context; -struct ggml_tensor; -struct ggml_cgraph; - -#ifdef __cplusplus -extern "C" { -#endif - -struct ggml_mpi_context; - -void ggml_mpi_backend_init(void); -void ggml_mpi_backend_free(void); - -struct ggml_mpi_context * ggml_mpi_init(void); -void ggml_mpi_free(struct ggml_mpi_context * ctx); - -int ggml_mpi_rank(struct ggml_mpi_context * ctx); - -void ggml_mpi_eval_init( - struct ggml_mpi_context * ctx_mpi, - int * n_tokens, - int * n_past, - int * n_threads); - -void ggml_mpi_graph_compute_pre( - struct ggml_mpi_context * ctx_mpi, - struct ggml_cgraph * gf, - int n_layers); - -void ggml_mpi_graph_compute_post( - struct ggml_mpi_context * ctx_mpi, - struct ggml_cgraph * gf, - int n_layers); - -#ifdef __cplusplus -} -#endif diff --git a/llama.cpp b/llama.cpp index 06ff4da61..102bc2020 100644 --- a/llama.cpp +++ b/llama.cpp @@ -26,9 +26,6 @@ #ifdef GGML_USE_METAL # include "ggml-metal.h" #endif -#ifdef GGML_USE_MPI -# include "ggml-mpi.h" -#endif #ifndef QK_K # ifdef GGML_QKK_64 # define QK_K 64 @@ -2270,10 +2267,6 @@ struct llama_context { // control vectors struct llama_control_vector cvec; - -#ifdef GGML_USE_MPI - ggml_mpi_context * ctx_mpi = NULL; -#endif }; static ggml_backend_buffer_type_t llama_default_buffer_type_offload(const llama_model & model, int gpu) { @@ -6336,10 +6329,7 @@ static struct ggml_tensor * llm_build_inp_embd( inpL = ggml_get_rows(ctx, tok_embd, lctx.inp_tokens); } else { -#ifdef GGML_USE_MPI - GGML_ASSERT(false && "not implemented"); -#endif - lctx.inp_embd = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, batch.n_tokens); + lctx.inp_embd = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, batch.n_tokens); inpL = lctx.inp_embd; ggml_set_input(lctx.inp_embd); } @@ -11351,11 +11341,6 @@ static void llama_graph_compute( llama_context & lctx, ggml_cgraph * gf, int n_threads) { -#ifdef GGML_USE_MPI - const int64_t n_layer = lctx.model.hparams.n_layer; - ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer); -#endif - #ifdef GGML_USE_METAL if (ggml_backend_is_metal(lctx.backend_metal)) { ggml_backend_metal_set_n_cb(lctx.backend_metal, n_threads); @@ -11370,10 +11355,6 @@ static void llama_graph_compute( ggml_backend_sched_graph_compute_async(lctx.sched, gf); // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched)); - -#ifdef GGML_USE_MPI - ggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer); -#endif } // decode a batch of tokens by evaluating the transformer @@ -11411,12 +11392,6 @@ static int llama_decode_internal( } lctx.n_queued_tokens += n_tokens_all; -#ifdef GGML_USE_MPI - // TODO: needs fix after #3228 - GGML_ASSERT(false && "not implemented"); - //ggml_mpi_eval_init(lctx.ctx_mpi, &n_tokens, &n_past, &n_threads); -#endif - auto & kv_self = lctx.kv_self; const int64_t n_embd = hparams.n_embd; @@ -15546,10 +15521,6 @@ void llama_backend_init(void) { struct ggml_context * ctx = ggml_init(params); ggml_free(ctx); } - -#ifdef GGML_USE_MPI - ggml_mpi_backend_init(); -#endif } void llama_numa_init(enum ggml_numa_strategy numa) { @@ -15559,9 +15530,6 @@ void llama_numa_init(enum ggml_numa_strategy numa) { } void llama_backend_free(void) { -#ifdef GGML_USE_MPI - ggml_mpi_backend_free(); -#endif ggml_quantize_free(); } @@ -15962,20 +15930,6 @@ struct llama_context * llama_new_context_with_model( } } -#ifdef GGML_USE_MPI - ctx->ctx_mpi = ggml_mpi_init(); - - if (ggml_mpi_rank(ctx->ctx_mpi) > 0) { - // Enter a blocking eval loop with dummy input, letting rank=0 drive the process - // TODO: needs fix after #3228 - GGML_ASSERT(false && "not implemented"); - //const std::vector tmp(ctx->model.hparams.n_ctx, llama_token_bos(ctx)); - //while (!llama_eval(ctx, tmp.data(), tmp.size(), 0, 0)) {}; - llama_backend_free(); - exit(1); - } -#endif - return ctx; } diff --git a/scripts/LlamaConfig.cmake.in b/scripts/LlamaConfig.cmake.in index f842c7137..92e39708b 100644 --- a/scripts/LlamaConfig.cmake.in +++ b/scripts/LlamaConfig.cmake.in @@ -5,7 +5,6 @@ set(LLAMA_SHARED_LIB @BUILD_SHARED_LIBS@) set(LLAMA_BLAS @LLAMA_BLAS@) set(LLAMA_CUDA @LLAMA_CUDA@) set(LLAMA_METAL @LLAMA_METAL@) -set(LLAMA_MPI @LLAMA_MPI@) set(LLAMA_CLBLAST @LLAMA_CLBLAST@) set(LLAMA_HIPBLAS @LLAMA_HIPBLAS@) set(LLAMA_ACCELERATE @LLAMA_ACCELERATE@) @@ -37,10 +36,6 @@ if (LLAMA_METAL) find_library(METALKIT_FRAMEWORK MetalKit REQUIRED) endif() -if (LLAMA_MPI) - find_package(MPI REQUIRED) -endif() - if (LLAMA_CLBLAST) find_package(CLBlast REQUIRED) endif() From 33c8d50accd6dca73c9c4af00a05e24209c160fe Mon Sep 17 00:00:00 2001 From: Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com> Date: Sun, 19 May 2024 19:18:39 -0700 Subject: [PATCH 080/137] Add provisions for windows support for BF16 code including CMake provision for enabling AVX512_BF16 (#7258) --- CMakeLists.txt | 8 ++++++++ ggml-impl.h | 12 ++++++++++++ ggml.c | 24 ++++++++++++++++-------- ggml.h | 1 + llama.cpp | 1 + 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 616698c7f..92c9f09eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,7 @@ option(LLAMA_AVX2 "llama: enable AVX2" option(LLAMA_AVX512 "llama: enable AVX512" OFF) option(LLAMA_AVX512_VBMI "llama: enable AVX512-VBMI" OFF) option(LLAMA_AVX512_VNNI "llama: enable AVX512-VNNI" OFF) +option(LLAMA_AVX512_BF16 "llama: enable AVX512-BF16" OFF) option(LLAMA_FMA "llama: enable FMA" ${INS_ENB}) # in MSVC F16C is implied with AVX2/AVX512 if (NOT MSVC) @@ -1060,6 +1061,10 @@ elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LW add_compile_definitions($<$:__AVX512VNNI__>) add_compile_definitions($<$:__AVX512VNNI__>) endif() + if (LLAMA_AVX512_BF16) + add_compile_definitions($<$:__AVX512BF16__>) + add_compile_definitions($<$:__AVX512BF16__>) + endif() elseif (LLAMA_AVX2) list(APPEND ARCH_FLAGS /arch:AVX2) elseif (LLAMA_AVX) @@ -1091,6 +1096,9 @@ elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LW if (LLAMA_AVX512_VNNI) list(APPEND ARCH_FLAGS -mavx512vnni) endif() + if (LLAMA_AVX512_BF16) + list(APPEND ARCH_FLAGS -mavx512bf16) + endif() endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64") message(STATUS "PowerPC detected") diff --git a/ggml-impl.h b/ggml-impl.h index 59684fa81..5ff014fe3 100644 --- a/ggml-impl.h +++ b/ggml-impl.h @@ -17,6 +17,18 @@ #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) +#if defined(_WIN32) + +#define m512bh(p) p +#define m512i(p) p + +#else + +#define m512bh(p) (__m512bh)(p) +#define m512i(p) (__m512i)(p) + +#endif + /** * Converts brain16 to float32. * diff --git a/ggml.c b/ggml.c index 3a104c486..53da231ee 100644 --- a/ggml.c +++ b/ggml.c @@ -406,10 +406,10 @@ void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) { int i = 0; #if defined(__AVX512BF16__) for (; i + 32 <= n; i += 32) { - _mm512_storeu_ps( - (__m512 *)(y + i), - (__m512)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16), - _mm512_loadu_ps(x + i))); + _mm512_storeu_si512( + (__m512i *)(y + i), + m512i(_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16), + _mm512_loadu_ps(x + i)))); } #endif for (; i < n; i++) { @@ -1666,10 +1666,10 @@ static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t __m512 c1 = _mm512_setzero_ps(); __m512 c2 = _mm512_setzero_ps(); for (; i + 64 <= n; i += 64) { - c1 = _mm512_dpbf16_ps(c1, (__m512bh)_mm512_loadu_ps((const float *)(x + i)), - (__m512bh)_mm512_loadu_ps((const float *)(y + i))); - c2 = _mm512_dpbf16_ps(c2, (__m512bh)_mm512_loadu_ps((const float *)(x + i + 32)), - (__m512bh)_mm512_loadu_ps((const float *)(y + i + 32))); + c1 = _mm512_dpbf16_ps(c1, m512bh(_mm512_loadu_si512((x + i))), + m512bh(_mm512_loadu_si512((y + i)))); + c2 = _mm512_dpbf16_ps(c2, m512bh(_mm512_loadu_si512((x + i + 32))), + m512bh(_mm512_loadu_si512((y + i + 32)))); } sumf += (ggml_float)_mm512_reduce_add_ps(c1); sumf += (ggml_float)_mm512_reduce_add_ps(c2); @@ -23137,6 +23137,14 @@ int ggml_cpu_has_avx512_vnni(void) { #endif } +int ggml_cpu_has_avx512_bf16(void) { +#if defined(__AVX512BF16__) + return 1; +#else + return 0; +#endif +} + int ggml_cpu_has_fma(void) { #if defined(__FMA__) return 1; diff --git a/ggml.h b/ggml.h index 8c13f4ba8..774757101 100644 --- a/ggml.h +++ b/ggml.h @@ -2390,6 +2390,7 @@ extern "C" { GGML_API int ggml_cpu_has_avx512 (void); GGML_API int ggml_cpu_has_avx512_vbmi(void); GGML_API int ggml_cpu_has_avx512_vnni(void); + GGML_API int ggml_cpu_has_avx512_bf16(void); GGML_API int ggml_cpu_has_fma (void); GGML_API int ggml_cpu_has_neon (void); GGML_API int ggml_cpu_has_arm_fma (void); diff --git a/llama.cpp b/llama.cpp index 102bc2020..ca3e9fcc0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -18074,6 +18074,7 @@ const char * llama_print_system_info(void) { s += "AVX512 = " + std::to_string(ggml_cpu_has_avx512()) + " | "; s += "AVX512_VBMI = " + std::to_string(ggml_cpu_has_avx512_vbmi()) + " | "; s += "AVX512_VNNI = " + std::to_string(ggml_cpu_has_avx512_vnni()) + " | "; + s += "AVX512_BF16 = " + std::to_string(ggml_cpu_has_avx512_bf16()) + " | "; s += "FMA = " + std::to_string(ggml_cpu_has_fma()) + " | "; s += "NEON = " + std::to_string(ggml_cpu_has_neon()) + " | "; s += "ARM_FMA = " + std::to_string(ggml_cpu_has_arm_fma()) + " | "; From 2789baf480ba7dc9281b65f601f0918e58920f54 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 20 May 2024 08:55:09 +0300 Subject: [PATCH 081/137] tests : fix --keep_split -> --keep-split (#7374) --- examples/quantize/tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/quantize/tests.sh b/examples/quantize/tests.sh index 160c12bee..a3ca74c68 100644 --- a/examples/quantize/tests.sh +++ b/examples/quantize/tests.sh @@ -41,8 +41,8 @@ $SPLIT --split-max-tensors 28 $WORK_PATH/gemma-1.1-2b-it.Q8_0.gguf $WORK_PATH/g echo PASS echo -# 3. Requant model with '--keep_split' -$QUANTIZE --allow-requantize --keep_split $WORK_PATH/ggml-model-split-00001-of-00006.gguf $WORK_PATH/ggml-model-requant.gguf Q4_K +# 3. Requant model with '--keep-split' +$QUANTIZE --allow-requantize --keep-split $WORK_PATH/ggml-model-split-00001-of-00006.gguf $WORK_PATH/ggml-model-requant.gguf Q4_K echo PASS echo @@ -51,7 +51,7 @@ $MAIN --model $WORK_PATH/ggml-model-requant-00001-of-00006.gguf --random-prompt echo PASS echo -# 4. Requant mode without '--keep_split' +# 4. Requant mode without '--keep-split' $QUANTIZE --allow-requantize $WORK_PATH/ggml-model-split-00001-of-00006.gguf $WORK_PATH/ggml-model-requant-merge.gguf Q4_K echo PASS echo From e932094d58f513d5996c3efc9f6fed8238894c57 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 20 May 2024 08:56:05 +0300 Subject: [PATCH 082/137] server : return error on too large embedding input (#7389) --- examples/server/server.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 7978f979d..6af5cb96e 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1981,8 +1981,7 @@ struct server_context { slot.state = SLOT_STATE_PROCESSING; slot.command = SLOT_COMMAND_NONE; slot.release(); - slot.print_timings(); - send_final_response(slot); + send_error(slot, "input is too large to process. increase the physical batch size", ERROR_TYPE_SERVER); continue; } } else { From 1cc0155d04918cb3017afa472acea51b77483c4a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 20 May 2024 10:16:41 +0300 Subject: [PATCH 083/137] server : tuning tests (#7388) * server : don't pass temperature as string * server : increase timeout * tests : fix the fix 0.8f -> 0.8 ggml-ci * tests : set explicit temperature --- examples/server/tests/features/results.feature | 2 ++ examples/server/tests/features/steps/steps.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/server/tests/features/results.feature b/examples/server/tests/features/results.feature index 5deb278c2..4ab8ad20c 100644 --- a/examples/server/tests/features/results.feature +++ b/examples/server/tests/features/results.feature @@ -13,6 +13,7 @@ Feature: Results Scenario Outline: consistent results with same seed Given slots + And 0.0 temperature Then the server is starting Then the server is healthy @@ -30,6 +31,7 @@ Feature: Results Scenario Outline: different results with different seed Given slots + And 1.0 temperature Then the server is starting Then the server is healthy diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 7da503f2c..26d9359d7 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -199,7 +199,7 @@ async def step_wait_for_the_server_to_be_started(context, expecting_status): case 'ready' | 'idle': await wait_for_health_status(context, context.base_url, 200, 'ok', - timeout=10, + timeout=30, params={'fail_on_no_slot': 0, 'include_slots': 0}, slots_idle=context.n_slots, slots_processing=0, @@ -883,7 +883,7 @@ async def request_completion(prompt, "cache_prompt": cache_prompt, "id_slot": id_slot, "seed": seed if seed is not None else 42, - "temperature": temperature if temperature is not None else "0.8f", + "temperature": temperature if temperature is not None else 0.8, "n_probs": 2, }, headers=headers, From 65c58207ece92ad213f4bfd0f91dcb2dfb664f5b Mon Sep 17 00:00:00 2001 From: junchao-loongson <68935141+junchao-loongson@users.noreply.github.com> Date: Mon, 20 May 2024 15:19:21 +0800 Subject: [PATCH 084/137] ggml : add loongarch lsx and lasx support (#6454) * add loongarch lsx and lasx optimize code * Add loongarch compilation support to makefile * revert stb_image.h * opt bytes_from_nibbles_32 and sum_i16_pairs_float * fix undeclared * format code * update * update 2 --------- Co-authored-by: Jinyang He --- CMakeLists.txt | 13 + Makefile | 5 + ggml-impl.h | 28 + ggml-quants.c | 2089 +++++++++++++++++++++++++++++++++++++++++++++++- ggml.c | 189 +++++ 5 files changed, 2316 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92c9f09eb..9cc60039a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,8 @@ set(LLAMA_SCHED_MAX_COPIES "4" CACHE STRING "llama: max input copies for pipeli option(LLAMA_BUILD_TESTS "llama: build tests" ${LLAMA_STANDALONE}) option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE}) option(LLAMA_BUILD_SERVER "llama: build server example" ON) +option(LLAMA_LASX "llama: enable lasx" ON) +option(LLAMA_LSX "llama: enable lsx" ON) # add perf arguments option(LLAMA_PERF "llama: enable perf" OFF) @@ -1108,6 +1110,17 @@ elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64") list(APPEND ARCH_FLAGS -mcpu=native -mtune=native) #TODO: Add targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be) endif() +elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64") + message(STATUS "loongarch64 detected") + + list(APPEND ARCH_FLAGS -march=loongarch64) + if (LLAMA_LASX) + list(APPEND ARCH_FLAGS -mlasx) + endif() + if (LLAMA_LSX) + list(APPEND ARCH_FLAGS -mlsx) + endif() + else() message(STATUS "Unknown architecture") endif() diff --git a/Makefile b/Makefile index 9a26aec50..6b7c853b3 100644 --- a/Makefile +++ b/Makefile @@ -379,6 +379,11 @@ ifneq ($(filter ppc64le%,$(UNAME_M)),) CUDA_POWER_ARCH = 1 endif +ifneq ($(filter loongarch64%,$(UNAME_M)),) + MK_CFLAGS += -mlasx + MK_CXXFLAGS += -mlasx +endif + else MK_CFLAGS += -march=rv64gcv -mabi=lp64d MK_CXXFLAGS += -march=rv64gcv -mabi=lp64d diff --git a/ggml-impl.h b/ggml-impl.h index 5ff014fe3..362d40f4d 100644 --- a/ggml-impl.h +++ b/ggml-impl.h @@ -455,6 +455,34 @@ static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) { #include #endif +#if defined(__loongarch64) +#if defined(__loongarch_asx) +#include +#endif +#if defined(__loongarch_sx) +#include +#endif +#endif + +#if defined(__loongarch_asx) + +typedef union { + int32_t i; + float f; +} ft_union; + +/* float type data load instructions */ +static __m128 __lsx_vreplfr2vr_s(float val) { + ft_union fi_tmpval = {.f = val}; + return (__m128)__lsx_vreplgr2vr_w(fi_tmpval.i); +} + +static __m256 __lasx_xvreplfr2vr_s(float val) { + ft_union fi_tmpval = {.f = val}; + return (__m256)__lasx_xvreplgr2vr_w(fi_tmpval.i); +} +#endif + #ifdef __F16C__ #ifdef _MSC_VER diff --git a/ggml-quants.c b/ggml-quants.c index ff1059863..ed40ca74a 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -262,6 +262,403 @@ static const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4 static const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4 #endif +#if defined(__loongarch_asx) + +#ifdef __clang__ +#define VREGS_PREFIX "$vr" +#define XREGS_PREFIX "$xr" +#else // GCC +#define VREGS_PREFIX "$f" +#define XREGS_PREFIX "$f" +#endif +#define __ALL_REGS "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31" +// Convert __m128i to __m256i +static inline __m256i ____m256i(__m128i in) { + __m256i out = __lasx_xvldi(0); + __asm__ volatile ( + ".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " XREGS_PREFIX"\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " VREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + : [out] "+f" (out) : [in] "f" (in) + ); + return out; +} +// Convert two __m128i to __m256i +static inline __m256i lasx_set_q(__m128i inhi, __m128i inlo) { + __m256i out; + __asm__ volatile ( + ".irp i," __ALL_REGS "\n\t" + " .ifc %[hi], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[lo], " VREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x20 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".ifnc %[out], %[hi] \n\t" + ".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " XREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[hi], " VREGS_PREFIX "\\j \n\t" + " xvori.b $xr\\i, $xr\\j, 0 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".endif \n\t" + : [out] "=f" (out), [hi] "+f" (inhi) + : [lo] "f" (inlo) + ); + return out; +} +// Convert __m256i low part to __m128i +static inline __m128i lasx_extracti128_lo(__m256i in) { + __m128i out; + __asm__ volatile ( + ".ifnc %[out], %[in] \n\t" + ".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " XREGS_PREFIX "\\j \n\t" + " vori.b $vr\\i, $vr\\j, 0 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + ".endif \n\t" + : [out] "=f" (out) : [in] "f" (in) + ); + return out; +} +// Convert __m256i high part to __m128i +static inline __m128i lasx_extracti128_hi(__m256i in) { + __m128i out; + __asm__ volatile ( + ".irp i," __ALL_REGS "\n\t" + " .ifc %[out], " VREGS_PREFIX "\\i \n\t" + " .irp j," __ALL_REGS "\n\t" + " .ifc %[in], " XREGS_PREFIX "\\j \n\t" + " xvpermi.q $xr\\i, $xr\\j, 0x11 \n\t" + " .endif \n\t" + " .endr \n\t" + " .endif \n\t" + ".endr \n\t" + : [out] "=f" (out) : [in] "f" (in) + ); + return out; +} + +static __m256i lasx_set_w(int e7, int e6, int e5, int e4, int e3, int e2, int e1, int e0) { + v8i32 __ret = {e0, e1, e2, e3, e4, e5, e6, e7}; + return (__m256i)__ret; +} + +static __m128i lsx_set_w(int32_t a, int32_t b, int32_t c, int32_t d) { + v4i32 __ret = {d, c, b, a}; + return (__m128i)__ret; +} + +static __m256i lasx_set_d(int64_t a, int64_t b, int64_t c, int64_t d) { + v4i64 __ret = {d, c, b, a}; + return (__m256i)__ret; +} + +static __m256i lasx_insertf128( __m128i x, __m128i y) { + return lasx_set_q(x, y); +} + +static __m128i lsx_shuffle_b(__m128i a, __m128i b) { + __m128i mask_f, zero, tmp0, tmp2, mask; + int f = 0x8f; + mask_f = __lsx_vreplgr2vr_b(f); + zero = __lsx_vldi(0); + tmp0 = __lsx_vand_v(b, mask_f); // get mask with low 4 bit and sign bits + tmp0 = __lsx_vori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive + mask = __lsx_vsle_b(zero, tmp0); // if mask >= 0, set mask + tmp2 = __lsx_vand_v(tmp0, mask); // maskout the in2 < ones + return __lsx_vshuf_b(a, zero, tmp2); +} + +static __m256i lasx_shuffle_b(__m256i a, __m256i b) { + __m256i mask_f, zero, tmp0, tmp2, mask; + int f = 0x8f; + mask_f = __lasx_xvreplgr2vr_b(f); + zero = __lasx_xvldi(0); + tmp0 = __lasx_xvand_v(b, mask_f); // get mask with low 4 bit and sign bits + tmp0 = __lasx_xvori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive + mask = __lasx_xvsle_b(zero, tmp0); // if mask >= 0, set mask + tmp2 = __lasx_xvand_v(tmp0, mask); // maskout the in2 < ones + return __lasx_xvshuf_b(a, zero, tmp2); +} + +static __m256i lasx_extu8_16(__m128i a) { + __m128i zero = __lsx_vldi(0); + __m128i vlo = __lsx_vilvl_b(zero, a); + __m128i vhi = __lsx_vilvh_b(zero, a); + return lasx_set_q(vhi, vlo); +} + +static __m256i lasx_ext8_16(__m128i a) { + __m128i sign = __lsx_vslti_b(a, 0); + __m128i vlo = __lsx_vilvl_b(sign, a); + __m128i vhi = __lsx_vilvh_b(sign, a); + return lasx_set_q(vhi, vlo); +} + +static __m256i lasx_ext16_32(__m128i a) { + __m256i tmp1; + tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 0), 0); + tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 1), 1); + tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 2), 2); + tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 3), 3); + tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 4), 4); + tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 5), 5); + tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 6), 6); + tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 7), 7); + return tmp1; +} + +static __m128i lasx_extracti128( __m256i a, int pos) { + __m128i ret; + if( pos == 0) + { + ret = lasx_extracti128_lo(a); + } else { + ret = lasx_extracti128_hi(a); + } + return ret; +} + +static __m128 lasx_extractf128( __m256 a, int pos) { + __m128 ret; + if( pos == 0) + { + ret = (__m128)lasx_extracti128_lo((__m256i)a); + } else { + ret = (__m128)lasx_extracti128_hi((__m256i)a); + } + return ret; +} + +static __m128i lsx_hadd_h(__m128i a, __m128i b) { + __m128i tmp1 = __lsx_vpickev_h(b, a); + __m128i tmp2 = __lsx_vpickod_h(b, a); + return __lsx_vadd_h(tmp1, tmp2); +} + +static __m128i lsx_hadd_w(__m128i a, __m128i b) { + __m128i tmp1 = __lsx_vpickev_w(b, a); + __m128i tmp2 = __lsx_vpickod_w(b, a); + return __lsx_vadd_w(tmp1, tmp2); +} + +static __m128 lsx_hadd_s(__m128 a, __m128 b) { + __m128 tmp1 = (__m128)__lsx_vpickev_w((__m128i)b, (__m128i)a); + __m128 tmp2 = (__m128)__lsx_vpickod_w((__m128i)b, (__m128i)a); + + return __lsx_vfadd_s(tmp1, tmp2); +} + +static __m256i lasx_maddubs_h(__m256i a, __m256i b) { + __m256i tmp1, tmp2; + tmp1 = __lasx_xvmulwev_h_b(a, b); + tmp2 = __lasx_xvmulwod_h_b(a, b); + return __lasx_xvsadd_h(tmp1, tmp2); +} + +static __m256i lasx_madd_h(__m256i a, __m256i b) { + __m256i tmp1, tmp2; + tmp1 = __lasx_xvmulwev_w_h(a, b); + tmp2 = __lasx_xvmulwod_w_h(a, b); + return __lasx_xvadd_w(tmp1, tmp2); +} + +static __m256i lasx_packs_w(__m256i a, __m256i b) { + __m256i tmp, tmp1; + tmp = __lasx_xvsat_w(a, 15); + tmp1 = __lasx_xvsat_w(b, 15); + return __lasx_xvpickev_h(tmp1, tmp); +} + +static __m256i lasx_packs_h(__m256i a, __m256i b) { + __m256i tmp, tmp1; + tmp = __lasx_xvsat_h(a, 7); + tmp1 = __lasx_xvsat_h(b, 7); + return __lasx_xvpickev_b(tmp1, tmp); +} + +static __m128i lsx_packs_w(__m128i a, __m128i b) { + __m128i tmp, tmp1; + tmp = __lsx_vsat_w(a, 15); + tmp1 = __lsx_vsat_w(b, 15); + return __lsx_vpickev_h(tmp1, tmp); +} + +static __m128i lsx_packs_h(__m128i a, __m128i b) { + __m128i tmp, tmp1; + tmp = __lsx_vsat_h(a, 7); + tmp1 = __lsx_vsat_h(b, 7); + return __lsx_vpickev_b(tmp1, tmp); +} + +static __m128i lsx_packus_h(__m128i a, __m128i b) { + __m128i tmp, tmp1; + tmp = __lsx_vsat_hu(a, 7); + tmp1 = __lsx_vsat_hu(b, 7); + return __lsx_vpickev_b(tmp1, tmp); +} + + +static __m128i lsx_maddubs_h(__m128i a, __m128i b) { + __m128i tmp1, tmp2; + tmp1 = __lsx_vmulwev_h_b(a, b); + tmp2 = __lsx_vmulwod_h_b(a, b); + return __lsx_vsadd_h(tmp1, tmp2); +} + +static __m128i lsx_madd_h(__m128i a, __m128i b) { + __m128i tmp1, tmp2; + tmp1 = __lsx_vmulwev_w_h(a, b); + tmp2 = __lsx_vmulwod_w_h(a, b); + return __lsx_vadd_w(tmp1, tmp2); +} + +// multiply int8_t, add results pairwise twice +static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) { + // Get absolute values of x vectors + const __m128i ax = __lsx_vsigncov_b(x, x); + // Sign the values of the y vectors + const __m128i sy = __lsx_vsigncov_b(x, y); + // Perform multiplication and create 16-bit values + const __m128i dot = lsx_maddubs_h(ax, sy); + const __m128i ones = __lsx_vreplgr2vr_h(1); + return lsx_madd_h(ones, dot); +} + +// horizontally add 8 floats +static inline float hsum_float_8(const __m256 x) { + __m128 res = lasx_extractf128(x, 1); + ft_union tmp; + res = __lsx_vfadd_s(res, lasx_extractf128(x, 0)); + res = __lsx_vfadd_s(res, (__m128)__lsx_vpickod_d((__m128i)res, (__m128i)res)); + res = __lsx_vfadd_s(res, (__m128)__lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w(res, 1), 0)); + tmp.i = __lsx_vpickve2gr_w(res, 0); + return tmp.f; +} + +// horizontally add 8 int32_t +static inline int hsum_i32_8(const __m256i a) { + + __m256i tmp1 = __lasx_xvpermi_q(a, a, 0x11); + __m256i tmp2 = __lasx_xvpermi_q(a, a, 0x00); + + __m128i tmp1_128 = lasx_extracti128_lo(tmp1); + __m128i tmp2_128 = lasx_extracti128_lo(tmp2); + + __m128i sum128 = __lsx_vadd_w(tmp1_128, tmp2_128); + + __m128i ev = __lsx_vpickev_w(sum128, sum128); + __m128i od = __lsx_vpickod_w(sum128, sum128); + __m128i sum64 = __lsx_vadd_w(ev, od); + + int sum64_1, sum64_2; + sum64_1 = __lsx_vpickve2gr_w(sum64, 0); + sum64_2 = __lsx_vpickve2gr_w(sum64, 1); + + return sum64_1 + sum64_2; +} + +// horizontally add 4 int32_t +static inline int hsum_i32_4(const __m128i a) { + __m128i ev = __lsx_vpickev_w(a, a); + __m128i od = __lsx_vpickod_w(a, a); + __m128i sum64 = __lsx_vadd_w(ev, od); + + int sum64_1, sum64_2; + sum64_1 = __lsx_vpickve2gr_w(sum64, 0); + sum64_2 = __lsx_vpickve2gr_w(sum64, 1); + + return sum64_1 + sum64_2; +} + +// spread 32 bits to 32 bytes { 0x00, 0xFF } +static inline __m256i bytes_from_bits_32(const uint8_t * x) { + + uint32_t x32; + memcpy(&x32, x, sizeof(uint32_t)); + const __m256i shuf_mask = lasx_set_d( + 0x0303030303030303, 0x0202020202020202, + 0x0101010101010101, 0x0000000000000000); + + __m256i bytes = lasx_shuffle_b(__lasx_xvreplgr2vr_w(x32), shuf_mask); + const __m256i bit_mask = __lasx_xvreplgr2vr_d(0x7fbfdfeff7fbfdfe); + bytes = __lasx_xvor_v(bytes, bit_mask); + return __lasx_xvseq_b(bytes, __lasx_xvreplgr2vr_d(-1)); +} + +// Unpack 32 4-bit fields into 32 bytes +// The output vector contains 32 bytes, each one in [ 0 .. 15 ] interval +static inline __m256i bytes_from_nibbles_32(const uint8_t * rsi) { + const __m128i lo = __lsx_vld((const __m128i *)rsi, 0); + __m128i hi = __lsx_vsrli_h(lo, 4); + return __lasx_xvandi_b(lasx_insertf128(hi, lo), 0xf); +} + +// add int16_t pairwise and return as float vector +static inline __m256 sum_i16_pairs_float(const __m256i x) { + __m256i v = __lasx_xvpackod_h(x, x); + __m256i summed_pairs = __lasx_xvaddwev_w_h(x, v); + return __lasx_xvffint_s_w(summed_pairs); +} + +static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) { + // Perform multiplication and create 16-bit values + const __m256i dot = lasx_maddubs_h(ax, sy); + return sum_i16_pairs_float(dot); +} + +// multiply int8_t, add results pairwise twice and return as float vector +static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) { + + // Get absolute values of x vectors + const __m256i ax = __lasx_xvsigncov_b(x, x); + // Sign the values of the y vectors + const __m256i sy = __lasx_xvsigncov_b(x, y); + + return mul_sum_us8_pairs_float(ax, sy); +} + +static inline __m128i packNibbles( __m256i bytes ) { + // Move bits within 16-bit lanes from 0000_abcd_0000_efgh into 0000_0000_abcd_efgh + const __m256i lowByte = __lasx_xvreplgr2vr_h(0xFF); + __m256i high = __lasx_xvandn_v(lowByte, bytes); + __m256i low = __lasx_xvand_v(lowByte, bytes); + high = __lasx_xvsrli_h(high, 4); + bytes = __lasx_xvor_v(low, high); + // Compress uint16_t lanes into bytes + __m128i *r0 = (__m128i *)&bytes; + __m256i tmp_h128 = __lasx_xvpermi_q(bytes, bytes, 0x11); + __m128i *r1 = (__m128i *)&tmp_h128; + + __m128i zero = __lsx_vldi(0); + __m128i tmp, tmp2, tmp3; + + tmp = __lsx_vmax_h(zero, *r0); + tmp2 = __lsx_vsat_hu(tmp, 7); + + tmp = __lsx_vmax_h(zero, *r1); + tmp3 = __lsx_vsat_hu(tmp, 7); + return __lsx_vpickev_b(tmp3, tmp2); +} +#endif //__loongarch_asx + // reference implementation for deterministic creation of model files void quantize_row_q4_0_reference(const float * restrict x, block_q4_0 * restrict y, int64_t k) { static const int qk = QK4_0; @@ -649,6 +1046,7 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) // store result __riscv_vse8_v_i8m1(y[i].qs , vs, vl); } + #elif defined(__POWER9_VECTOR__) for (int i = 0; i < nb; i++) { vector float srcv [8]; @@ -680,6 +1078,69 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) } vec_xst(vec_pack(vec_pack(vi[0], vi[1]), vec_pack(vi[2], vi[3])), 0, &y[i].qs[0]); vec_xst(vec_pack(vec_pack(vi[4], vi[5]), vec_pack(vi[6], vi[7])), 16, &y[i].qs[0]); + +#elif defined(__loongarch_asx) + for (int i = 0; i < nb; i++) { + ft_union fi; + __m256 v0 = (__m256)__lasx_xvld( x , 0); + __m256 v1 = (__m256)__lasx_xvld( x , 32); + __m256 v2 = (__m256)__lasx_xvld( x , 64); + __m256 v3 = (__m256)__lasx_xvld( x , 96); + x += 32; + + // Compute max(abs(e)) for the block + const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f ); + __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 ); + max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) ); + max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) ); + max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) ); + + __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs , 0) ); + max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) ); + __m128 tmp = max4; + max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vinsgr2vr_w(tmp, __lsx_vpickve2gr_w( max4, 1 ), 0 )); + fi.i = __lsx_vpickve2gr_w( (__m128i)max4, 0 ); + const float max_scalar = fi.f; + + // Quantize these floats + const float d = max_scalar / 127.f; + y[i].d = GGML_FP32_TO_FP16(d); + const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f; + const __m256 mul = (__m256)__lasx_xvreplfr2vr_s( id ); + + // Apply the multiplier + v0 = __lasx_xvfmul_s( v0, mul ); + v1 = __lasx_xvfmul_s( v1, mul ); + v2 = __lasx_xvfmul_s( v2, mul ); + v3 = __lasx_xvfmul_s( v3, mul ); + + // Round to nearest integer + __m256i i0 = __lasx_xvftintrne_w_s( v0 ); + __m256i i1 = __lasx_xvftintrne_w_s( v1 ); + __m256i i2 = __lasx_xvftintrne_w_s( v2 ); + __m256i i3 = __lasx_xvftintrne_w_s( v3 ); + + __m128i ni0 = lasx_extracti128( i0, 0 ); + __m128i ni1 = lasx_extracti128( i0, 1); + __m128i ni2 = lasx_extracti128( i1, 0); + __m128i ni3 = lasx_extracti128( i1, 1); + __m128i ni4 = lasx_extracti128( i2, 0); + __m128i ni5 = lasx_extracti128( i2, 1); + __m128i ni6 = lasx_extracti128( i3, 0); + __m128i ni7 = lasx_extracti128( i3, 1); + + // Convert int32 to int16 + ni0 = lsx_packs_w( ni0, ni1 ); + ni2 = lsx_packs_w( ni2, ni3 ); + ni4 = lsx_packs_w( ni4, ni5 ); + ni6 = lsx_packs_w( ni6, ni7 ); + // Convert int16 to int8 + ni0 = lsx_packs_h( ni0, ni2 ); + ni4 = lsx_packs_h( ni4, ni6 ); + + __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0); + __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0); + } #else GGML_UNUSED(nb); @@ -828,12 +1289,12 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 ), _mm256_castps256_ps128( maxAbs ) ); max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); - const float maxScalar = _mm_cvtss_f32( max4 ); + const float max_scalar = _mm_cvtss_f32( max4 ); // Quantize these floats - const float d = maxScalar / 127.f; + const float d = max_scalar / 127.f; y[i].d = GGML_FP32_TO_FP16(d); - const float id = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; + const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f; const __m256 mul = _mm256_set1_ps( id ); // Apply the multiplier @@ -936,6 +1397,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) int sum = __riscv_vmv_x_s_i16m1_i16(vwrs); y[i].s = GGML_FP32_TO_FP16(sum*d); } + #elif defined(__POWER9_VECTOR__) for (int i = 0; i < nb; i++) { vector float srcv [8]; @@ -975,6 +1437,73 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) accv = vec_add(accv, vec_sld(accv, accv, 4)); accv = vec_add(accv, vec_sld(accv, accv, 8)); y[i].s = GGML_FP32_TO_FP16(d * vec_extract(accv, 0)); + +#elif defined(__loongarch_asx) + for (int i = 0; i < nb; i++) { + ft_union ft; + __m256 v0 = (__m256)__lasx_xvld( x , 0 ); + __m256 v1 = (__m256)__lasx_xvld( x , 32 ); + __m256 v2 = (__m256)__lasx_xvld( x , 64 ); + __m256 v3 = (__m256)__lasx_xvld( x , 96 ); + x += 32; + + // Compute max(abs(e)) for the block + const __m256 sign_bit = __lasx_xvreplfr2vr_s( -0.0f ); + __m256 max_abs = (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v0 ); + max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v1 ) ); + max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v2 ) ); + max_abs = __lasx_xvfmax_s( max_abs, (__m256)__lasx_xvandn_v( (__m256i)sign_bit, (__m256i)v3 ) ); + + __m128 max4 = __lsx_vfmax_s( lasx_extractf128( max_abs, 1 ), lasx_extractf128( max_abs, 0) ); + max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) ); + __m128 tmp = max4; + max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vextrins_w((__m128i)tmp, (__m128i)max4, 0x10 )); + ft.i = __lsx_vpickve2gr_w( (__m128i)max4, 0 ); + const float max_scalar = ft.f; + + // Quantize these floats + const float d = max_scalar / 127.f; + y[i].d = GGML_FP32_TO_FP16(d); + const float id = ( max_scalar != 0.0f ) ? 127.f / max_scalar : 0.0f; + const __m256 mul = __lasx_xvreplfr2vr_s( id ); + + // Apply the multiplier + v0 = __lasx_xvfmul_s( v0, mul ); + v1 = __lasx_xvfmul_s( v1, mul ); + v2 = __lasx_xvfmul_s( v2, mul ); + v3 = __lasx_xvfmul_s( v3, mul ); + + // Round to nearest integer + __m256i i0 = __lasx_xvftintrne_w_s( v0 ); + __m256i i1 = __lasx_xvftintrne_w_s( v1 ); + __m256i i2 = __lasx_xvftintrne_w_s( v2 ); + __m256i i3 = __lasx_xvftintrne_w_s( v3 ); + + __m128i ni0 = lasx_extracti128(i0, 0); + __m128i ni1 = lasx_extracti128( i0, 1); + __m128i ni2 = lasx_extracti128( i1, 0); + __m128i ni3 = lasx_extracti128( i1, 1); + __m128i ni4 = lasx_extracti128( i2, 0 ); + __m128i ni5 = lasx_extracti128( i2, 1); + __m128i ni6 = lasx_extracti128( i3, 0); + __m128i ni7 = lasx_extracti128( i3, 1); + + // Compute the sum of the quants and set y[i].s + const __m128i s0 = __lsx_vadd_w(__lsx_vadd_w(ni0, ni1), __lsx_vadd_w(ni2, ni3)); + const __m128i s1 = __lsx_vadd_w(__lsx_vadd_w(ni4, ni5), __lsx_vadd_w(ni6, ni7)); + y[i].s = GGML_FP32_TO_FP16(d * hsum_i32_4(__lsx_vadd_w(s0, s1))); + + // Convert int32 to int16 + ni0 = lsx_packs_w( ni0, ni1 ); + ni2 = lsx_packs_w( ni2, ni3 ); + ni4 = lsx_packs_w( ni4, ni5 ); + ni6 = lsx_packs_w( ni6, ni7 ); + // Convert int16 to int8 + ni0 = lsx_packs_h( ni0, ni2 ); + ni4 = lsx_packs_h( ni4, ni6 ); + + __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0); + __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0); } #else GGML_UNUSED(nb); @@ -3470,6 +3999,43 @@ static inline __m128i get_scale_shuffle(int i) { }; return _mm_loadu_si128((const __m128i*)k_shuffle + i); } +#elif defined(__loongarch_asx) +// shuffles to pick the required scales in dot products +static inline __m256i get_scale_shuffle_q3k(int i) { + static const uint8_t k_shuffle[128] = { + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, + 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, + 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11, + 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15, + }; + return __lasx_xvld((const __m256i*)k_shuffle + i, 0); +} +static inline __m256i get_scale_shuffle_k4(int i) { + static const uint8_t k_shuffle[256] = { + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, + 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, + 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, + 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, + 10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11,10,11, + 12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13,12,13, + 14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15,14,15 + }; + return __lasx_xvld((const __m256i*)k_shuffle + i, 0); +} +static inline __m128i get_scale_shuffle(int i) { + static const uint8_t k_shuffle[128] = { + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, + 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, + 10,10,10,10,10,10,10,10, 11,11,11,11,11,11,11,11, + 12,12,12,12,12,12,12,12, 13,13,13,13,13,13,13,13, + 14,14,14,14,14,14,14,14, 15,15,15,15,15,15,15,15 + }; + return __lsx_vld((const __m128i*)k_shuffle + i, 0); +} #endif void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { @@ -3819,6 +4385,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r } *s = sumf; + #elif defined(__POWER9_VECTOR__) const vector signed char lowMask = vec_splats((signed char)0xF); const vector unsigned char v4 = vec_splats((unsigned char)0x4); @@ -3859,6 +4426,149 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + // Initialize accumulator with zeros + __m256 acc = (__m256)__lasx_xvldi(0); + + // Main loop + for (int i = 0; i < nb; ++i) { + /* Compute combined scale for the block */ + const __m256 d = __lasx_xvreplfr2vr_s( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) ); + + __m256i qx = bytes_from_nibbles_32(x[i].qs); + + // Now we have a vector with bytes in [ 0 .. 15 ] interval. Offset them into [ -8 .. +7 ] interval. + const __m256i off = __lasx_xvreplgr2vr_b( 8 ); + qx = __lasx_xvsub_b( qx, off ); + + __m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0); + + const __m256 q = mul_sum_i8_pairs_float(qx, qy); + + /* Multiply q with scale and accumulate */ + acc = __lasx_xvfmadd_s( d, q, acc ); + } + + *s = hsum_float_8(acc); +#elif defined(__loongarch_sx) + // set constants + const __m128i low_mask = __lsx_vreplgr2vr_b(0xF); + const __m128i off = __lsx_vreplgr2vr_b(8); + + // Initialize accumulator with zeros + __m128 acc_0 = __lsx_vldi(0); + __m128 acc_1 = __lsx_vldi(0); + __m128 acc_2 = __lsx_vldi(0); + __m128 acc_3 = __lsx_vldi(0); + + // First round without accumulation + { + _mm_prefetch(&x[0] + sizeof(block_q4_0), _MM_HINT_T0); + _mm_prefetch(&y[0] + sizeof(block_q8_0), _MM_HINT_T0); + + // Compute combined scale for the block 0 and 1 + const __m128 d_0_1 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[0].d) * GGML_FP16_TO_FP32(y[0].d) ); + + const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[0].qs, 0); + + __m128i bx_0 = __lsx_vand_v(low_mask, tmp_0_1); + __m128i by_0 = __lsx_vld((const __m128i *)y[0].qs, 0); + bx_0 = __lsx_vsub_b(bx_0, off); + const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0); + + __m128i bx_1 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_0_1, 4)); + __m128i by_1 = __lsx_vld((const __m128i *)(y[0].qs + 16), 0); + bx_1 = __lsx_vsub_b(bx_1, off); + const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1); + + // Compute combined scale for the block 2 and 3 + const __m128 d_2_3 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[1].d) * GGML_FP16_TO_FP32(y[1].d) ); + + const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[1].qs, 0); + + __m128i bx_2 = __lsx_vand_v(low_mask, tmp_2_3); + __m128i by_2 = __lsx_vld((const __m128i *)y[1].qs, 0); + bx_2 = __lsx_vsub_b(bx_2, off); + const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2); + + __m128i bx_3 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_2_3, 4)); + __m128i by_3 = __lsx_vld((const __m128i *)(y[1].qs + 16), 0); + bx_3 = __lsx_vsub_b(bx_3, off); + const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3); + + // Convert int32_t to float + __m128 p0 = __lsx_vffint_s_w(i32_0); + __m128 p1 = __lsx_vffint_s_w(i32_1); + __m128 p2 = __lsx_vffint_s_w(i32_2); + __m128 p3 = __lsx_vffint_s_w(i32_3); + + // Apply the scale + acc_0 = __lsx_vfmul_s( d_0_1, p0 ); + acc_1 = __lsx_vfmul_s( d_0_1, p1 ); + acc_2 = __lsx_vfmul_s( d_2_3, p2 ); + acc_3 = __lsx_vfmul_s( d_2_3, p3 ); + } + + assert(nb % 2 == 0); // TODO: handle odd nb + + // Main loop + for (int i = 2; i < nb; i+=2) { + + // Compute combined scale for the block 0 and 1 + const __m128 d_0_1 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d) ); + + const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[i].qs, 0); + + __m128i bx_0 = __lsx_vand_v(low_mask, tmp_0_1); + __m128i by_0 = __lsx_vld((const __m128i *)y[i].qs, 0); + bx_0 = __lsx_vsub_b(bx_0, off); + const __m128i i32_0 = mul_sum_i8_pairs(bx_0, by_0); + + __m128i bx_1 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_0_1, 4)); + __m128i by_1 = __lsx_vld((const __m128i *)(y[i].qs + 16), 0); + bx_1 = __lsx_vsub_b(bx_1, off); + const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1); + + //_mm_prefetch(&x[i] + 2 * sizeof(block_q4_0), _MM_HINT_T0); + //_mm_prefetch(&y[i] + 2 * sizeof(block_q8_0), _MM_HINT_T0); + + // Compute combined scale for the block 2 and 3 + const __m128 d_2_3 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[i + 1].d) * GGML_FP16_TO_FP32(y[i + 1].d) ); + + const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[i + 1].qs, 0); + + __m128i bx_2 = __lsx_vand_v(low_mask, tmp_2_3); + __m128i by_2 = __lsx_vld((const __m128i *)y[i + 1].qs, 0); + bx_2 = __lsx_vsub_b(bx_2, off); + const __m128i i32_2 = mul_sum_i8_pairs(bx_2, by_2); + + __m128i bx_3 = __lsx_vand_v(low_mask, __lsx_vsrli_d(tmp_2_3, 4)); + __m128i by_3 = __lsx_vld((const __m128i *)(y[i + 1].qs + 16), 0); + bx_3 = __lsx_vsub_b(bx_3, off); + const __m128i i32_3 = mul_sum_i8_pairs(bx_3, by_3); + + // Convert int32_t to float + __m128 p0 = __lsx_vffint_s_w(i32_0); + __m128 p1 = __lsx_vffint_s_w(i32_1); + __m128 p2 = __lsx_vffint_s_w(i32_2); + __m128 p3 = __lsx_vffint_s_w(i32_3); + + // Apply the scale + __m128 p0_d = __lsx_vfmul_s( d_0_1, p0 ); + __m128 p1_d = __lsx_vfmul_s( d_0_1, p1 ); + __m128 p2_d = __lsx_vfmul_s( d_2_3, p2 ); + __m128 p3_d = __lsx_vfmul_s( d_2_3, p3 ); + + // Acummulate + acc_0 = __lsx_vfadd_s(p0_d, acc_0); + acc_1 = __lsx_vfadd_s(p1_d, acc_1); + acc_2 = __lsx_vfadd_s(p2_d, acc_2); + acc_3 = __lsx_vfadd_s(p3_d, acc_3); + } + + *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3); + #else // scalar float sumf = 0.0; @@ -4078,6 +4788,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r } *s = sumf; + #elif defined(__POWER9_VECTOR__) const vector signed char lowMask = vec_splats((signed char)0xF); const vector unsigned char v4 = vec_splats((unsigned char)0x4); @@ -4118,6 +4829,38 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + // Initialize accumulator with zeros + __m256 acc = (__m256)__lasx_xvldi(0); + + float summs = 0; + + // Main loop + for (int i = 0; i < nb; ++i) { + const float d0 = GGML_FP16_TO_FP32(x[i].d); + const float d1 = GGML_FP16_TO_FP32(y[i].d); + + summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s); + + const __m256 d0v = __lasx_xvreplfr2vr_s( d0 ); + const __m256 d1v = __lasx_xvreplfr2vr_s( d1 ); + + // Compute combined scales + const __m256 d0d1 = __lasx_xvfmul_s( d0v, d1v ); + + // Load 16 bytes, and unpack 4 bit fields into bytes, making 32 bytes + const __m256i qx = bytes_from_nibbles_32(x[i].qs); + const __m256i qy = __lasx_xvld( (const __m256i *)y[i].qs, 0); + + const __m256 xy = mul_sum_us8_pairs_float(qx, qy); + + // Accumulate d0*d1*x*y + acc = __lasx_xvfmadd_s( d0d1, xy, acc ); + } + + *s = hsum_float_8(acc) + summs; + #else // scalar float sumf = 0.0; @@ -4403,6 +5146,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r } *s = sumf; + #elif defined(__POWER9_VECTOR__) const vector signed char lowMask = vec_splats((signed char)0xF); const vector unsigned char v4 = vec_splats((unsigned char)4); @@ -4446,6 +5190,31 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * restrict s, size_t bs, const void * r vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + // Initialize accumulator with zeros + __m256 acc = (__m256)__lasx_xvldi(0); + + // Main loop + for (int i = 0; i < nb; i++) { + /* Compute combined scale for the block */ + const __m256 d = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d)); //FIXME + + __m256i qx = bytes_from_nibbles_32(x[i].qs); + __m256i bxhi = bytes_from_bits_32(x[i].qh); + bxhi = __lasx_xvandn_v(bxhi, __lasx_xvreplgr2vr_b((char)0xF0)); + qx = __lasx_xvor_v(qx, bxhi); + + __m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0); + + const __m256 q = mul_sum_i8_pairs_float(qx, qy); + + /* Multiply q with scale and accumulate */ + acc = __lasx_xvfmadd_s(d, q, acc); + } + + *s = hsum_float_8(acc); + #else // scalar float sumf = 0.0; @@ -4750,6 +5519,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r } *s = sumf; + #elif defined(__POWER9_VECTOR__) const vector signed char lowMask = vec_splats((signed char)0xF); const vector unsigned char v4 = vec_splats((unsigned char)0x4); @@ -4797,6 +5567,34 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * restrict s, size_t bs, const void * r vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + // Initialize accumulator with zeros + __m256 acc = (__m256)__lasx_xvldi(0); + + float summs = 0.0f; + + // Main loop + for (int i = 0; i < nb; i++) { + const __m256 dx = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[i].d)); + + summs += GGML_FP16_TO_FP32(x[i].m) * GGML_FP16_TO_FP32(y[i].s); + + __m256i qx = bytes_from_nibbles_32(x[i].qs); + __m256i bxhi = bytes_from_bits_32(x[i].qh); + bxhi = __lasx_xvand_v(bxhi, __lasx_xvreplgr2vr_b(0x10)); + qx = __lasx_xvor_v(qx, bxhi); + + const __m256 dy = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[i].d)); + const __m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0); + + const __m256 q = mul_sum_us8_pairs_float(qx, qy); + + acc = __lasx_xvfmadd_s(q, __lasx_xvfmul_s(dx, dy), acc); + } + + *s = hsum_float_8(acc) + summs; + #else // scalar float sumf = 0.0; @@ -4973,6 +5771,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r } *s = sumf; + #elif defined(__POWER9_VECTOR__) vector float vsumf0 = vec_splats(0.0f); @@ -5012,6 +5811,26 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + // Initialize accumulator with zeros + __m256 acc = (__m256)__lasx_xvldi(0); + + // Main loop + for (int i = 0; i < nb; ++i) { + // Compute combined scale for the block + const __m256 d = __lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[i].d) * GGML_FP16_TO_FP32(y[i].d)); + __m256i qx = __lasx_xvld((const __m256i *)x[i].qs, 0); + __m256i qy = __lasx_xvld((const __m256i *)y[i].qs, 0); + + const __m256 q = mul_sum_i8_pairs_float(qx, qy); + + // Multiply q with scale and accumulate + acc = __lasx_xvfmadd_s( d, q, acc ); + } + + *s = hsum_float_8(acc); + #else // scalar float sumf = 0.0; @@ -5416,8 +6235,6 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r vector signed int vsumi6 = vec_splats((int32_t)0); vector signed int vsumi7 = vec_splats((int32_t)0); - const uint8_t * restrict q2 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; for (int j = 0; j < QK_K/128; ++j) { __builtin_prefetch(q2, 0, 1); @@ -5508,6 +6325,71 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); +#elif defined __loongarch_asx + + const __m256i m3 = __lasx_xvreplgr2vr_b(3); + const __m128i m4 = __lsx_vreplgr2vr_b(0xF); + + __m256 acc = (__m256)__lasx_xvldi(0); + + for (int i = 0; i < nb; ++i) { + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + + const uint8_t * restrict q2 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + const __m128i mins_and_scales = __lsx_vld((const __m128i*)x[i].scales, 0); + const __m128i scales8 = __lsx_vand_v(mins_and_scales, m4); + const __m128i mins8 = __lsx_vand_v(__lsx_vsrli_h(mins_and_scales, 4), m4); + const __m256i mins = lasx_ext8_16(mins8); + const __m256i prod = lasx_madd_h(mins, __lasx_xvld((const __m256i*)y[i].bsums, 0)); + + acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(dmin), __lasx_xvffint_s_w(prod), acc); + + const __m256i all_scales = lasx_ext8_16(scales8); + const __m128i l_scales = lasx_extracti128(all_scales, 0); + const __m128i h_scales = lasx_extracti128(all_scales, 1); + const __m256i scales[2] = {lasx_insertf128(l_scales, l_scales), lasx_insertf128(h_scales, h_scales)}; + + __m256i sumi = __lasx_xvldi(0); + + for (int j = 0; j < QK_K/128; ++j) { + + const __m256i q2bits = __lasx_xvld((const __m256i*)q2, 0); q2 += 32; + + const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + + const __m256i q2_0 = __lasx_xvand_v(q2bits, m3); + const __m256i q2_1 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 2), m3); + const __m256i q2_2 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 4), m3); + const __m256i q2_3 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 6), m3); + + __m256i p0 = lasx_maddubs_h(q2_0, q8_0); + __m256i p1 = lasx_maddubs_h(q2_1, q8_1); + __m256i p2 = lasx_maddubs_h(q2_2, q8_2); + __m256i p3 = lasx_maddubs_h(q2_3, q8_3); + + p0 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(0)), p0); + p1 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(1)), p1); + p2 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(2)), p2); + p3 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(3)), p3); + + p0 = __lasx_xvadd_w(p0, p1); + p2 = __lasx_xvadd_w(p2, p3); + + sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p0, p2)); + } + + acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc); + + } + + *s = hsum_float_8(acc); + #else float sumf = 0; @@ -5778,6 +6660,7 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; + #elif defined(__POWER9_VECTOR__) const vector signed char lowMask = vec_splats((signed char)0x3); const vector signed char lowScaleMask = vec_splats((signed char)0xF); @@ -5859,6 +6742,63 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); +#elif defined __loongarch_asx + + const __m256i m3 = __lasx_xvreplgr2vr_b(3); + + __m256 acc = (__m256)__lasx_xvldi(0); + + uint32_t ud, um; + const uint8_t * restrict db = (const uint8_t *)&ud; + const uint8_t * restrict mb = (const uint8_t *)&um; + + float summs = 0; + + // TODO: optimize this + + for (int i = 0; i < nb; ++i) { + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + + const uint8_t * restrict q2 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + const uint32_t * restrict sc = (const uint32_t *)x[i].scales; + ud = (sc[0] >> 0) & 0x0f0f0f0f; + um = (sc[0] >> 4) & 0x0f0f0f0f; + + int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3]; + summs += dmin * smin; + + const __m128i q2bits = __lsx_vld((const __m128i*)q2, 0); + const __m256i q2_0 = __lasx_xvand_v(lasx_insertf128(__lsx_vsrli_h(q2bits, 2), q2bits), m3); + const __m256i q2_1 = __lasx_xvand_v(lasx_insertf128(__lsx_vsrli_h(q2bits, 6), __lsx_vsrli_h(q2bits, 4)), m3); + + const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0); + const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0); + + const __m256i p0 = lasx_maddubs_h(q2_0, q8_0); + const __m256i p1 = lasx_maddubs_h(q2_1, q8_1); + + const __m256i p_0 = lasx_ext16_32(lasx_extracti128(p0, 0)); + const __m256i p_1 = lasx_ext16_32(lasx_extracti128(p0, 1)); + const __m256i p_2 = lasx_ext16_32(lasx_extracti128(p1, 0)); + const __m256i p_3 = lasx_ext16_32(lasx_extracti128(p1, 1)); + + ft_union t0, t1, t2, t3; + t0.f = d * db[0]; + t1.f = d * db[1]; + t2.f = d * db[2]; + t3.f = d * db[3]; + acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t0.i), __lasx_xvffint_s_w(p_0), acc); + acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t1.i), __lasx_xvffint_s_w(p_1), acc); + acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t2.i), __lasx_xvffint_s_w(p_2), acc); + acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t3.i), __lasx_xvffint_s_w(p_3), acc); + } + + *s = hsum_float_8(acc) + summs; + #else float sumf = 0; @@ -6396,6 +7336,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r vector signed int vsumi6 = vec_splats((int32_t)0); vector signed int vsumi7 = vec_splats((int32_t)0); + const uint8_t * restrict q3 = x[i].qs; const int8_t * restrict q8 = y[i].qs; @@ -6507,6 +7448,107 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined __loongarch_asx + + const __m256i m3 = __lasx_xvreplgr2vr_b(3); + const __m256i mone = __lasx_xvreplgr2vr_b(1); + const __m128i m32 = __lsx_vreplgr2vr_b(32); + + __m256 acc = (__m256)__lasx_xvldi(0); + + uint32_t aux[3]; + + for (int i = 0; i < nb; ++i) { + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + // Set up scales + memcpy(aux, x[i].scales, 12); + __m128i scales128 = lsx_set_w( + ((aux[1] >> 4) & kmask2) | (((aux[2] >> 6) & kmask1) << 4), + ((aux[0] >> 4) & kmask2) | (((aux[2] >> 4) & kmask1) << 4), + (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4), + (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4)); + scales128 = __lsx_vsub_b(scales128, m32); + const __m256i all_scales = lasx_ext8_16(scales128); + const __m128i l_scales = lasx_extracti128(all_scales, 0); + const __m128i h_scales = lasx_extracti128(all_scales, 1); + const __m256i scales[2] = {lasx_insertf128(l_scales, l_scales), lasx_insertf128(h_scales, h_scales)}; + + // high bit + const __m256i hbits = __lasx_xvld((const __m256i*)x[i].hmask, 0); + + // integer accumulator + __m256i sumi = __lasx_xvldi(0); + + int bit = 0; + int is = 0; + + const uint8_t * restrict q3 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + for (int j = 0; j < QK_K/128; ++j) { + // load low 2 bits + const __m256i q3bits = __lasx_xvld((const __m256i*)q3, 0); q3 += 32; + + // prepare low and high bits + const __m256i q3l_0 = __lasx_xvand_v(q3bits, m3); + const __m256i q3h_0 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvandn_v(hbits, __lasx_xvslli_h(mone, bit)), bit), 2); + ++bit; + + const __m256i q3l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 2), m3); + const __m256i q3h_1 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvandn_v(hbits, __lasx_xvslli_h(mone, bit)), bit), 2); + ++bit; + + const __m256i q3l_2 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 4), m3); + const __m256i q3h_2 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvandn_v(hbits, __lasx_xvslli_h(mone, bit)), bit), 2); + ++bit; + + const __m256i q3l_3 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 6), m3); + const __m256i q3h_3 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvandn_v(hbits, __lasx_xvslli_h(mone, bit)), bit), 2); + ++bit; + + // load Q8 quants + const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + + // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use lasx_maddubs_h, + // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set, + // and 2 if the high bit was set) + __m256i q8s_0 = lasx_maddubs_h(q3h_0, q8_0); + __m256i q8s_1 = lasx_maddubs_h(q3h_1, q8_1); + __m256i q8s_2 = lasx_maddubs_h(q3h_2, q8_2); + __m256i q8s_3 = lasx_maddubs_h(q3h_3, q8_3); + + __m256i p16_0 = lasx_maddubs_h(q3l_0, q8_0); + __m256i p16_1 = lasx_maddubs_h(q3l_1, q8_1); + __m256i p16_2 = lasx_maddubs_h(q3l_2, q8_2); + __m256i p16_3 = lasx_maddubs_h(q3l_3, q8_3); + + p16_0 = __lasx_xvsub_h(p16_0, q8s_0); + p16_1 = __lasx_xvsub_h(p16_1, q8s_1); + p16_2 = __lasx_xvsub_h(p16_2, q8s_2); + p16_3 = __lasx_xvsub_h(p16_3, q8s_3); + + // multiply with scales + p16_0 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0); + p16_1 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1); + p16_2 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2); + p16_3 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3); + + // accumulate + p16_0 = __lasx_xvadd_w(p16_0, p16_1); + p16_2 = __lasx_xvadd_w(p16_2, p16_3); + sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_2)); + } + // multiply with block scale and accumulate + acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);//FIXME + } + + *s = hsum_float_8(acc); + #else // scalar version // This function is written like this so the compiler can manage to vectorize most of it @@ -6962,6 +8004,73 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined __loongarch_asx + + const __m256i m3 = __lasx_xvreplgr2vr_b(3); + const __m256i m1 = __lasx_xvreplgr2vr_b(1); + + __m256 acc = (__m256)__lasx_xvldi(0); + + uint64_t aux64; + + uint16_t aux16[2]; + const int8_t * aux8 = (const int8_t *)aux16; + + for (int i = 0; i < nb; ++i) { + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + + const uint8_t * restrict q3 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + const __m256i scale_0 = lasx_insertf128(__lasx_xvreplgr2vr_h(aux8[2] - 8), __lasx_xvreplgr2vr_h(aux8[0] - 8)); + const __m256i scale_1 = lasx_insertf128(__lasx_xvreplgr2vr_h(aux8[3] - 8), __lasx_xvreplgr2vr_h(aux8[1] - 8)); + + memcpy(&aux64, x[i].hmask, 8); + + __m128i haux = __lsx_vinsgr2vr_d(haux, aux64, 0); + haux = __lsx_vinsgr2vr_d(haux, aux64 >> 1, 1); + __m256i q3h_0 = lasx_insertf128(__lsx_vsrli_h(haux, 2), haux); + __m256i q3h_1 = __lasx_xvsrli_h(q3h_0, 4); + q3h_0 = __lasx_xvslli_h(__lasx_xvandn_v(q3h_0, m1), 2); + q3h_1 = __lasx_xvslli_h(__lasx_xvandn_v(q3h_1, m1), 2); + + // load low 2 bits + const __m128i q3bits = __lsx_vld((const __m128i*)q3, 0); + + // prepare low and high bits + const __m256i q3aux = lasx_insertf128(__lsx_vsrli_h(q3bits, 2), q3bits); + const __m256i q3l_0 = __lasx_xvand_v(q3aux, m3); + const __m256i q3l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q3aux, 4), m3); + + // load Q8 quants + const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0); + const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0); + + // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use lasx_maddubs_h, + // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set, + // and 2 if the high bit was set) + const __m256i q8s_0 = lasx_maddubs_h(q3h_0, q8_0); + const __m256i q8s_1 = lasx_maddubs_h(q3h_1, q8_1); + + __m256i p16_0 = lasx_maddubs_h(q3l_0, q8_0); + __m256i p16_1 = lasx_maddubs_h(q3l_1, q8_1); + + p16_0 = __lasx_xvsub_h(p16_0, q8s_0); + p16_1 = __lasx_xvsub_h(p16_1, q8s_1); + + // multiply with scales + p16_0 = lasx_madd_h(scale_0, p16_0); + p16_1 = lasx_madd_h(scale_1, p16_1); + + p16_0 = __lasx_xvadd_w(p16_0, p16_1); + + // multiply with block scale and accumulate + acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(p16_0), acc); + } + + *s = hsum_float_8(acc); + #else int8_t aux8[QK_K]; @@ -7450,8 +8559,69 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); -#else +#elif defined __loongarch_asx + const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); + + __m256 acc = (__m256)__lasx_xvldi(0); + __m128 acc_m = (__m128)__lsx_vldi(0); + + for (int i = 0; i < nb; ++i) { + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + + memcpy(utmp, x[i].scales, 12); + + const uint8_t * restrict q4 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0])); + + const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0); + const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1)); + const __m128i prod = lsx_madd_h(lasx_extracti128(mins_and_scales, 1), q8s); + acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m); + + const __m128i sc128 = lasx_extracti128(mins_and_scales, 0); + const __m256i scales = lasx_insertf128(sc128, sc128); + + __m256i sumi = __lasx_xvldi(0); + + for (int j = 0; j < QK_K/64; ++j) { + + const __m256i scale_l = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+0)); + const __m256i scale_h = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+1)); + + const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); q4 += 32; + const __m256i q4l = __lasx_xvand_v(q4bits, m4); + const __m256i q4h = __lasx_xvand_v(__lasx_xvsrli_h(q4bits, 4), m4); + + const __m256i q8l = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + __m256i p16l = lasx_maddubs_h(q4l, q8l); + p16l = lasx_madd_h(scale_l, p16l); + + const __m256i q8h = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + __m256i p16h = lasx_maddubs_h(q4h, q8h); + p16h = lasx_madd_h(scale_h, p16h); + const __m256i sumj = __lasx_xvadd_w(p16l, p16h); + + sumi = __lasx_xvadd_w(sumi, sumj); + } + + __m256 vd = __lasx_xvreplfr2vr_s(d); + acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc); + } + + acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vpermi_w((__m128i)acc_m, (__m128i)acc_m, 0xee)); + __m128i tmp1 = __lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w((__m128i)acc_m, 1), 0); + acc_m = __lsx_vfadd_s(acc_m, (__m128)tmp1); + + ft_union fi; + fi.i = __lsx_vpickve2gr_w(acc_m, 0); + *s = hsum_float_8(acc) + fi.f ; + +#else const uint8_t * scales = (const uint8_t*)&utmp[0]; const uint8_t * mins = (const uint8_t*)&utmp[2]; @@ -7797,6 +8967,51 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); +#elif defined __loongarch_asx + + const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); + + __m256 acc = (__m256)__lasx_xvldi(0); + + float summs = 0; + + uint16_t aux16[2]; + const uint8_t * scales = (const uint8_t *)aux16; + + for (int i = 0; i < nb; ++i) { + + const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d; + const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d; + const __m256 vd = __lasx_xvreplfr2vr_s(d); + + const uint16_t * a = (const uint16_t *)x[i].scales; + aux16[0] = a[0] & 0x0f0f; + aux16[1] = (a[0] >> 4) & 0x0f0f; + + summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3])); + + const uint8_t * restrict q4 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); + const __m256i q4l = __lasx_xvand_v(q4bits, m4); + const __m256i q4h = __lasx_xvand_v(__lasx_xvsrli_h(q4bits, 4), m4); + + const __m256i q8l = __lasx_xvld((const __m256i*)(q8+ 0), 0); + const __m256i q8h = __lasx_xvld((const __m256i*)(q8+32), 0); + + const __m256i p16l = lasx_maddubs_h(q4l, q8l); + const __m256i p16h = lasx_maddubs_h(q4h, q8h); + + const __m256i p32l = lasx_madd_h(__lasx_xvreplgr2vr_h(scales[0]), p16l); + acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(p32l), acc); + + const __m256i p32h = lasx_madd_h(__lasx_xvreplgr2vr_h(scales[1]), p16h); + acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(p32h), acc); + } + + *s = hsum_float_8(acc) - summs; + #else uint8_t aux8[QK_K]; @@ -8322,6 +9537,84 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); +#elif defined __loongarch_asx + + const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); + const __m128i mzero = __lsx_vldi(0); + const __m256i mone = __lasx_xvreplgr2vr_b(1); + + __m256 acc = (__m256)__lasx_xvldi(0); + + float summs = 0.f; + + for (int i = 0; i < nb; ++i) { + + const uint8_t * restrict q5 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + +#if QK_K == 256 + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + + memcpy(utmp, x[i].scales, 12); +#else + // TODO + const float d = 0, dmin = 0; +#endif + + const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0])); + + const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0); + const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1)); + const __m128i prod = lsx_madd_h(lasx_extracti128(mins_and_scales, 1), q8s); + const __m128i hsum = lsx_hadd_w(lsx_hadd_w(prod, mzero), mzero); + summs += dmin * __lsx_vpickve2gr_w(hsum, 0); //TODO check + + const __m128i sc128 = lasx_extracti128(mins_and_scales, 0); + const __m256i scales = lasx_insertf128(sc128, sc128); + + const __m256i hbits = __lasx_xvld((const __m256i*)x[i].qh, 0); + __m256i hmask = mone; + + __m256i sumi = __lasx_xvldi(0); + + int bit = 0; + + for (int j = 0; j < QK_K/64; ++j) { + + const __m256i scale_0 = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+0)); + const __m256i scale_1 = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+1)); + + const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); q5 += 32; + + const __m256i q5l_0 = __lasx_xvand_v(q5bits, m4); + const __m256i q5h_0 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvand_v(hbits, hmask), bit++), 4); + const __m256i q5_0 = __lasx_xvadd_b(q5l_0, q5h_0); + hmask = __lasx_xvslli_h(hmask, 1); + + const __m256i q5l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q5bits, 4), m4); + const __m256i q5h_1 = __lasx_xvslli_h(__lasx_xvsrli_h(__lasx_xvand_v(hbits, hmask), bit++), 4); + const __m256i q5_1 = __lasx_xvadd_b(q5l_1, q5h_1); + hmask = __lasx_xvslli_h(hmask, 1); + + const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + + __m256i p16_0 = lasx_maddubs_h(q5_0, q8_0); + __m256i p16_1 = lasx_maddubs_h(q5_1, q8_1); + + p16_0 = lasx_madd_h(scale_0, p16_0); + p16_1 = lasx_madd_h(scale_1, p16_1); + + sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1)); + } + + __m256 vd = __lasx_xvreplfr2vr_s(d); + acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(sumi), acc); + } + + *s = hsum_float_8(acc) + summs; + #else const uint8_t * scales = (const uint8_t*)&utmp[0]; @@ -8696,6 +9989,52 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); +#elif defined __loongarch_asx + + const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); + const __m256i mone = __lasx_xvreplgr2vr_b(1); + + __m256 acc = (__m256)__lasx_xvldi(0); + + for (int i = 0; i < nb; ++i) { + + const uint8_t * restrict q5 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + + const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); + + const __m256i scale_l = lasx_insertf128(__lsx_vreplgr2vr_h(x[i].scales[1]), __lsx_vreplgr2vr_h(x[i].scales[0])); + const __m256i scale_h = lasx_insertf128(__lsx_vreplgr2vr_h(x[i].scales[3]), __lsx_vreplgr2vr_h(x[i].scales[2])); + + int64_t aux64; + memcpy(&aux64, x[i].qh, 8); + __m128i haux128 = __lsx_vinsgr2vr_d(haux128, aux64, 0); + haux128 = __lsx_vinsgr2vr_d(haux128, aux64 >> 1, 1); + const __m256i haux256 = lasx_insertf128(__lsx_vsrli_h(haux128, 2), haux128); + + const __m256i q5h_0 = __lasx_xvslli_h(__lasx_xvandn_v(haux256, mone), 4); + const __m256i q5h_1 = __lasx_xvslli_h(__lasx_xvandn_v(__lasx_xvsrli_h(haux256, 4), mone), 4); + + const __m256i q5l_0 = __lasx_xvand_v(q5bits, m4); + const __m256i q5l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q5bits, 4), m4); + + const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0); + const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0); + + const __m256i p16_0 = lasx_madd_h(scale_l, lasx_maddubs_h(q5l_0, q8_0)); + const __m256i p16_1 = lasx_madd_h(scale_h, lasx_maddubs_h(q5l_1, q8_1)); + const __m256i s16_0 = lasx_madd_h(scale_l, lasx_maddubs_h(q5h_0, q8_0)); + const __m256i s16_1 = lasx_madd_h(scale_h, lasx_maddubs_h(q5h_1, q8_1)); + + const __m256i dot = __lasx_xvsub_w(__lasx_xvadd_w(p16_0, p16_1), __lasx_xvadd_w(s16_0, s16_1)); + + acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(dot), acc); + } + + *s = hsum_float_8(acc); + #else int8_t aux8[QK_K]; @@ -9271,6 +10610,84 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); +#elif defined __loongarch_asx + + const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); + const __m256i m2 = __lasx_xvreplgr2vr_b(3); + const __m256i m32s = __lasx_xvreplgr2vr_b(32); + + __m256 acc = (__m256)__lasx_xvldi(0); + + for (int i = 0; i < nb; ++i) { + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + + const uint8_t * restrict q4 = x[i].ql; + const uint8_t * restrict qh = x[i].qh; + const int8_t * restrict q8 = y[i].qs; + + const __m128i scales = __lsx_vld((const __m128i*)x[i].scales, 0); + + __m256i sumi = __lasx_xvldi(0); + + int is = 0; + + for (int j = 0; j < QK_K/128; ++j) { + + const __m128i scale_0 = lsx_shuffle_b(scales, get_scale_shuffle(is + 0)); + const __m128i scale_1 = lsx_shuffle_b(scales, get_scale_shuffle(is + 1)); + const __m128i scale_2 = lsx_shuffle_b(scales, get_scale_shuffle(is + 2)); + const __m128i scale_3 = lsx_shuffle_b(scales, get_scale_shuffle(is + 3)); + is += 4; + + const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32; + const __m256i q4bits2 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32; + const __m256i q4bitsH = __lasx_xvld((const __m256i*)qh, 0); qh += 32; + + const __m256i q4h_0 = __lasx_xvslli_h(__lasx_xvand_v(q4bitsH, m2), 4); + const __m256i q4h_1 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 2), m2), 4); + const __m256i q4h_2 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 4), m2), 4); + const __m256i q4h_3 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 6), m2), 4); + + const __m256i q4_0 = __lasx_xvor_v(__lasx_xvand_v(q4bits1, m4), q4h_0); + const __m256i q4_1 = __lasx_xvor_v(__lasx_xvand_v(q4bits2, m4), q4h_1); + const __m256i q4_2 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits1, 4), m4), q4h_2); + const __m256i q4_3 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits2, 4), m4), q4h_3); + + const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + + __m256i q8s_0 = lasx_maddubs_h(m32s, q8_0); + __m256i q8s_1 = lasx_maddubs_h(m32s, q8_1); + __m256i q8s_2 = lasx_maddubs_h(m32s, q8_2); + __m256i q8s_3 = lasx_maddubs_h(m32s, q8_3); + + __m256i p16_0 = lasx_maddubs_h(q4_0, q8_0); + __m256i p16_1 = lasx_maddubs_h(q4_1, q8_1); + __m256i p16_2 = lasx_maddubs_h(q4_2, q8_2); + __m256i p16_3 = lasx_maddubs_h(q4_3, q8_3); + + p16_0 = __lasx_xvsub_h(p16_0, q8s_0); + p16_1 = __lasx_xvsub_h(p16_1, q8s_1); + p16_2 = __lasx_xvsub_h(p16_2, q8s_2); + p16_3 = __lasx_xvsub_h(p16_3, q8s_3); + + p16_0 = lasx_madd_h(lasx_ext8_16(scale_0), p16_0); + p16_1 = lasx_madd_h(lasx_ext8_16(scale_1), p16_1); + p16_2 = lasx_madd_h(lasx_ext8_16(scale_2), p16_2); + p16_3 = lasx_madd_h(lasx_ext8_16(scale_3), p16_3); + + sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1)); + sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_2, p16_3)); + } + + acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc); + } + + *s = hsum_float_8(acc); + #else int8_t aux8[QK_K]; @@ -9656,6 +11073,65 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); +#elif defined __loongarch_asx + + const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); + const __m256i m2 = __lasx_xvreplgr2vr_b(3); + const __m256i m32s = __lasx_xvreplgr2vr_b(32); + + __m256 acc = (__m256)__lasx_xvldi(0); + + for (int i = 0; i < nb; ++i) { + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + + const uint8_t * restrict q4 = x[i].ql; + const uint8_t * restrict qh = x[i].qh; + const int8_t * restrict q8 = y[i].qs; + + const __m64 scales_1 = __lasx_xvreplgr2vr_b(x[i].scales[0]); + const __m64 scales_2 = __lasx_xvreplgr2vr_b(x[i].scales[1]); + const __m64 scales_3 = __lasx_xvreplgr2vr_b(x[i].scales[2]); + const __m64 scales_4 = __lasx_xvreplgr2vr_b(x[i].scales[3]); + + __m256i sumi = __lasx_xvldi(0); + + __m128i scale_0 = __lsx_vinsgr2vr_d(scale_0, scales_1, 0); + scale_0 = __lsx_vinsgr2vr_d(scale_0, scales_2, 1); + __m128i scale_1 = __lsx_vinsgr2vr_d(scale_1, scales_3, 0); + scale_1 = __lsx_vinsgr2vr_d(scale_1, scales_4, 1); + + const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); + const __m128i q4bitsH = __lsx_vld((const __m128i*)qh, 0); + + const __m256i q4h_0 = __lasx_xvslli_h(__lasx_xvand_v(lasx_insertf128(__lasx_xvsrli_h(q4bitsH, 2), q4bitsH), m2), 4); + const __m256i q4h_1 = __lasx_xvslli_h(__lasx_xvand_v(lasx_insertf128(__lasx_xvsrli_h(q4bitsH, 6), __lasx_xvsrli_h(q4bitsH, 4)), m2), 4); + + const __m256i q4_0 = __lasx_xvor_v(__lasx_xvand_v(q4bits1, m4), q4h_0); + const __m256i q4_1 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits1, 4), m4), q4h_1); + + const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0); + const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0); + + __m256i q8s_0 = lasx_maddubs_h(m32s, q8_0); + __m256i q8s_1 = lasx_maddubs_h(m32s, q8_1); + + __m256i p16_0 = lasx_maddubs_h(q4_0, q8_0); + __m256i p16_1 = lasx_maddubs_h(q4_1, q8_1); + + p16_0 = __lasx_xvsub_h(p16_0, q8s_0); + p16_1 = __lasx_xvsub_h(p16_1, q8s_1); + + p16_0 = lasx_madd_h(lasx_ext8_16(scale_0), p16_0); + p16_1 = lasx_madd_h(lasx_ext8_16(scale_1), p16_1); + + sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1)); + + acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc); + } + + *s = hsum_float_8(acc); + #else int8_t aux8[QK_K]; @@ -9697,7 +11173,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r #endif -#if defined (__AVX2__) || defined (__ARM_NEON) || defined (__POWER9_VECTOR__) +#if defined (__AVX2__) || defined (__ARM_NEON) || defined (__POWER9_VECTOR__) || defined(__loongarch_asx) static const int8_t keven_signs_q2xs[1024] = { 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1, @@ -9927,6 +11403,49 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = 0.125f * vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + + uint32_t aux32[4]; + const uint8_t * aux8 = (const uint8_t *)aux32; + + __m256 accumf = (__m256)__lasx_xvldi(0); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint16_t * restrict q2 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + __m256i sumi1 = __lasx_xvldi(0); + __m256i sumi2 = __lasx_xvldi(0); + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + memcpy(aux32, q2, 4*sizeof(uint32_t)); q2 += 8; + + const __m256i q2_1 = lasx_set_d(iq2xxs_grid[aux8[ 3]], iq2xxs_grid[aux8[ 2]], iq2xxs_grid[aux8[1]], iq2xxs_grid[aux8[0]]); + const __m256i q2_2 = lasx_set_d(iq2xxs_grid[aux8[11]], iq2xxs_grid[aux8[10]], iq2xxs_grid[aux8[9]], iq2xxs_grid[aux8[8]]); + const __m256i s2_1 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127], + signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]); + const __m256i s2_2 = lasx_set_d(signs64[(aux32[3] >> 21) & 127], signs64[(aux32[3] >> 14) & 127], + signs64[(aux32[3] >> 7) & 127], signs64[(aux32[3] >> 0) & 127]); + const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1); + const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2); + const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); + const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); + const uint16_t ls1 = aux32[1] >> 28; + const uint16_t ls2 = aux32[3] >> 28; + const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1)); + const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1)); + sumi1 = __lasx_xvadd_w(sumi1, p1); + sumi2 = __lasx_xvadd_w(sumi2, p2); + } + + accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf); + } + + *s = 0.125f * hsum_float_8(accumf); + #else uint32_t aux32[2]; @@ -10201,6 +11720,181 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * *s = 0.125f * hsum_float_8(accumf); #endif +#elif defined(__loongarch_asx) + + const __m256i mone = __lasx_xvreplgr2vr_b(1); + static const char block_sign_shuffle_mask_1[32] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, + 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, + }; + static const char block_sign_shuffle_mask_2[32] = { + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0e, + }; + static const uint8_t bit_selector_mask_bytes[32] = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + }; + + const __m256i bit_selector_mask = __lasx_xvld((const __m256i*)bit_selector_mask_bytes, 0); + const __m256i block_sign_shuffle_1 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_1, 0); + const __m256i block_sign_shuffle_2 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_2, 0); + +#if QK_K == 64 + static const uint8_t k_bit_helper[16] = { + 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, + }; + const __m128i bit_helper = __lsx_vld((const __m128i*)k_bit_helper, 0); + const __m128i m511 = __lsx_vreplgr2vr_h(511); + typedef union { + __m128i vec_index; + uint16_t index[8]; + } index_t; + + index_t idx; + __m256 accumf = (__m256)__lasx_xvldi(0); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const __m128i q2_data = __lsx_vld((const __m128i*)x[i].qs, 0); + idx.vec_index = __lsx_vand_v(q2_data, m511); + + const __m128i partial_sign_bits = __lsx_vsrli_h(q2_data, 9); + const __m128i partial_sign_bits_upper = __lsx_vsrli_h(q2_data, 13); + const __m128i partial_sign_bits_for_counting = __lsx_vxor_v(partial_sign_bits, partial_sign_bits_upper); + + const __m128i odd_bits = lsx_shuffle_b(bit_helper, partial_sign_bits_for_counting); + const __m128i full_sign_bits = __lsx_vor_v(partial_sign_bits, odd_bits); + const __m256i full_signs = lasx_insertf128(full_sign_bits, full_sign_bits); + + const __m256i q8_1 = __lasx_xvld((const __m256i *)y[i].qs, 0); + const __m256i q8_2 = __lasx_xvld((const __m256i *)(y[i].qs+32), 0); + + const __m256i q2_1 = lasx_set_d(iq2xs_grid[idx.index[3]], iq2xs_grid[idx.index[2]], + iq2xs_grid[idx.index[1]], iq2xs_grid[idx.index[0]]); + const __m256i q2_2 = lasx_set_d(iq2xs_grid[idx.index[7]], iq2xs_grid[idx.index[6]], + iq2xs_grid[idx.index[5]], iq2xs_grid[idx.index[4]]); + __m256i signs; + signs = lasx_shuffle_b(full_signs, block_sign_shuffle_1); + signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask); + const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1); + + signs = lasx_shuffle_b(full_signs, block_sign_shuffle_2); + signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask); + const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2); + + const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); + const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); + + const __m256i sc1 = lasx_insertf128(_mm_set1_epi16(2*(x[i].scales[0] >> 4)+1), __lsx_vreplgr2vr_h(2*(x[i].scales[0] & 0xf)+1)); + const __m256i sc2 = lasx_insertf128(_mm_set1_epi16(2*(x[i].scales[1] >> 4)+1), __lsx_vreplgr2vr_h(2*(x[i].scales[1] & 0xf)+1)); + + const __m256i sum = __lasx_xvadd_w(lasx_madd_h(sc1, dot1), lasx_madd_h(sc2, dot2)); + + accumf = __lasx_vfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sum), accumf); + } + + *s = 0.125f * hsum_float_8(accumf); +#else + + static const uint8_t k_bit_helper[32] = { + 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, + 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, + }; + const __m256i bit_helper = __lasx_xvld((const __m256i*)k_bit_helper, 0); + const __m256i m511 = __lasx_xvreplgr2vr_h(511); + const __m128i m4 = __lsx_vreplgr2vr_b(0xf); + const __m128i m1 = __lsx_vreplgr2vr_b(1); + + uint64_t aux64; + + // somewhat hacky, but gives a significant boost in performance + __m256i aux_gindex; + const uint16_t * gindex = (const uint16_t *)&aux_gindex; + + __m256 accumf = (__m256)__lasx_xvldi(0); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint16_t * restrict q2 = x[i].qs; + const int8_t * restrict q8 = y[i].qs; + + memcpy(&aux64, x[i].scales, 8); + __m128i stmp = __lsx_vreplgr2vr_d(aux64); + stmp = __lsx_vilvl_b( __lsx_vand_v(__lsx_vsrli_h(stmp, 4), m4), __lsx_vand_v(stmp, m4)); + const __m128i scales = __lsx_vadd_b(__lsx_vslli_h(stmp, 1), m1); + + __m256i sumi1 = __lasx_xvldi(0); + __m256i sumi2 = __lasx_xvldi(0); + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 4) { + + const __m256i q2_data = __lasx_xvld((const __m256i*)q2, 0); q2 += 16; + aux_gindex = __lasx_xvand_v(q2_data, m511); + + const __m256i partial_sign_bits = __lasx_xvsrli_h(q2_data, 9); + const __m256i partial_sign_bits_upper = __lasx_xvsrli_h(q2_data, 13); + const __m256i partial_sign_bits_for_counting = __lasx_xvxor_v(partial_sign_bits, partial_sign_bits_upper); + + const __m256i odd_bits = lasx_shuffle_b(bit_helper, partial_sign_bits_for_counting); + const __m256i full_sign_bits = __lasx_xvor_v(partial_sign_bits, odd_bits); + + const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q8_3 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q8_4 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + + const __m256i q2_1 = lasx_set_d(iq2xs_grid[gindex[ 3]], iq2xs_grid[gindex[ 2]], + iq2xs_grid[gindex[ 1]], iq2xs_grid[gindex[ 0]]); + const __m256i q2_2 = lasx_set_d(iq2xs_grid[gindex[ 7]], iq2xs_grid[gindex[ 6]], + iq2xs_grid[gindex[ 5]], iq2xs_grid[gindex[ 4]]); + const __m256i q2_3 = lasx_set_d(iq2xs_grid[gindex[11]], iq2xs_grid[gindex[10]], + iq2xs_grid[gindex[ 9]], iq2xs_grid[gindex[ 8]]); + const __m256i q2_4 = lasx_set_d(iq2xs_grid[gindex[15]], iq2xs_grid[gindex[14]], + iq2xs_grid[gindex[13]], iq2xs_grid[gindex[12]]); + + const __m128i full_signs_l = lasx_extracti128(full_sign_bits, 0); + const __m128i full_signs_h = lasx_extracti128(full_sign_bits, 1); + const __m256i full_signs_1 = lasx_insertf128(full_signs_l, full_signs_l); + const __m256i full_signs_2 = lasx_insertf128(full_signs_h, full_signs_h); + + __m256i signs; + signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_1); + signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask); + const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1); + + signs = lasx_shuffle_b(full_signs_1, block_sign_shuffle_2); + signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask); + const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2); + + signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_1); + signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask); + const __m256i q8s_3 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_3); + + signs = lasx_shuffle_b(full_signs_2, block_sign_shuffle_2); + signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask); + const __m256i q8s_4 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_4); + + const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); + const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); + const __m256i dot3 = lasx_maddubs_h(q2_3, q8s_3); + const __m256i dot4 = lasx_maddubs_h(q2_4, q8s_4); + + const __m256i sc1 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+0))); + const __m256i sc2 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+1))); + const __m256i sc3 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+2))); + const __m256i sc4 = lasx_ext8_16(lsx_shuffle_b(scales, get_scale_shuffle(ib32+3))); + + sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot1, sc1)); + sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot2, sc2)); + sumi1 = __lasx_xvadd_w(sumi1, lasx_madd_h(dot3, sc3)); + sumi2 = __lasx_xvadd_w(sumi2, lasx_madd_h(dot4, sc4)); + } + + accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf); + + } + + *s = 0.125f * hsum_float_8(accumf); +#endif + #elif defined(__POWER9_VECTOR__) vector float vsumf0 = vec_splats(0.0f); @@ -10618,6 +12312,81 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * restrict s, size_t bs, const void * vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = 0.125f * vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 + }; + + static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + }; + + + const __m128i m4 = __lsx_vreplgr2vr_b(0xf); + const __m128i m1 = __lsx_vreplgr2vr_b(1); + + const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0); + const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0); + uint64_t aux64; + + __m256 accumf = (__m256)__lasx_xvldi(0); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint8_t * restrict qs = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint16_t * restrict signs = (const uint16_t *)(x[i].qs + QK_K/8); + const int8_t * restrict q8 = y[i].qs; + + __m128i tmp1; + memcpy(&aux64, x[i].scales, 8); + tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64, 0); + tmp1 = __lsx_vinsgr2vr_d(tmp1, aux64 >> 4, 1); + const __m128i scales8 = __lsx_vadd_b(__lsx_vslli_h(__lsx_vand_v(tmp1, m4), 1), m1); + const __m256i scales16 = lasx_ext8_16(scales8); // 0 2 4 6 8 10 12 14 1 3 5 7 9 11 13 15 + + __m256i sumi1 = __lasx_xvldi(0); + __m256i sumi2 = __lasx_xvldi(0); + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q2_1 = lasx_set_d(iq2s_grid[qs[3] | ((qh[ib32+0] << 2) & 0x300)], + iq2s_grid[qs[2] | ((qh[ib32+0] << 4) & 0x300)], + iq2s_grid[qs[1] | ((qh[ib32+0] << 6) & 0x300)], + iq2s_grid[qs[0] | ((qh[ib32+0] << 8) & 0x300)]); + const __m256i q2_2 = lasx_set_d(iq2s_grid[qs[7] | ((qh[ib32+1] << 2) & 0x300)], + iq2s_grid[qs[6] | ((qh[ib32+1] << 4) & 0x300)], + iq2s_grid[qs[5] | ((qh[ib32+1] << 6) & 0x300)], + iq2s_grid[qs[4] | ((qh[ib32+1] << 8) & 0x300)]); + qs += 8; + + __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | ((uint32_t) signs[1] << 16)); + aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2); + const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2); + const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1); + + aux256 = __lasx_xvreplgr2vr_w(signs[2] | ((uint32_t) signs[3] << 16)); + aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2); + const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2); + const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2); + + signs += 4; + + const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); // blocks 2*ib32+0, 2*ib32+1 + const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); // blocks 2*ib32+2, 2*ib32+3 + + const __m256i p1 = lasx_madd_h(dot1, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+0))); + const __m256i p2 = lasx_madd_h(dot2, lasx_shuffle_b(scales16, get_scale_shuffle_k4(ib32+1))); + sumi1 = __lasx_xvadd_w(sumi1, p1); + sumi2 = __lasx_xvadd_w(sumi2, p2); + } + + accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf); + } + + *s = 0.125f * hsum_float_8(accumf); + #else float sumf = 0; @@ -10857,6 +12626,54 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * restrict s, size_t bs, const void vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = 0.25f * vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + + const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; + + uint32_t aux32[2]; + + __m256 accumf = (__m256)__lasx_xvldi(0); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint8_t * restrict q3 = x[i].qs; + const uint8_t * restrict gas = x[i].qs + QK_K/4; + const int8_t * restrict q8 = y[i].qs; + __m256i sumi1 = __lasx_xvldi(0); + __m256i sumi2 = __lasx_xvldi(0); + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q2_1 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]], + iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]); + q3 += 8; + const __m256i q2_2 = lasx_set_w(iq3xxs_grid[q3[7]], iq3xxs_grid[q3[6]], iq3xxs_grid[q3[5]], iq3xxs_grid[q3[4]], + iq3xxs_grid[q3[3]], iq3xxs_grid[q3[2]], iq3xxs_grid[q3[1]], iq3xxs_grid[q3[0]]); + q3 += 8; + memcpy(aux32, gas, 8); gas += 8; + + const __m256i s2_1 = lasx_set_d(signs64[(aux32[0] >> 21) & 127], signs64[(aux32[0] >> 14) & 127], + signs64[(aux32[0] >> 7) & 127], signs64[(aux32[0] >> 0) & 127]); + const __m256i s2_2 = lasx_set_d(signs64[(aux32[1] >> 21) & 127], signs64[(aux32[1] >> 14) & 127], + signs64[(aux32[1] >> 7) & 127], signs64[(aux32[1] >> 0) & 127]); + const __m256i q8s_1 = __lasx_xvsigncov_b(s2_1, q8_1); + const __m256i q8s_2 = __lasx_xvsigncov_b(s2_2, q8_2); + const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); + const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); + const uint16_t ls1 = aux32[0] >> 28; + const uint16_t ls2 = aux32[1] >> 28; + + const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1)); + const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1)); + sumi1 = __lasx_xvadd_w(sumi1, p1); + sumi2 = __lasx_xvadd_w(sumi2, p2); + } + + accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf); + } + + *s = 0.25f * hsum_float_8(accumf); + #else uint32_t aux32; @@ -11202,6 +13019,89 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + + static const uint8_t k_mask1[32] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 + }; + + static const uint8_t k_mask2[32] = {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + }; + + const __m256i mask1 = __lasx_xvld((const __m256i*)k_mask1, 0); + const __m256i mask2 = __lasx_xvld((const __m256i*)k_mask2, 0); + + __m256i idx_shift = lasx_set_w(1, 2, 3, 4, 5, 6, 7, 8); + const __m256i idx_mask = __lasx_xvreplgr2vr_w(256); + + typedef union { + __m256i vec[2]; + uint32_t index[16]; + } index_t; + + index_t idx; + + __m256 accumf = (__m256)__lasx_xvldi(0); + for (int i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; + const uint8_t * restrict qs = x[i].qs; + const uint8_t * restrict qh = x[i].qh; + const uint16_t * restrict signs = (const uint16_t *)x[i].signs; + const int8_t * restrict q8 = y[i].qs; + __m256i sumi1 = __lasx_xvldi(0); + __m256i sumi2 = __lasx_xvldi(0); + for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { + const __m256i q8_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q8_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i idx_l = lasx_extu8_16(__lsx_vld(qs, 0)); qs += 16; + idx.vec[0] = __lasx_xvreplgr2vr_w(qh[ib32+0]); + idx.vec[1] = __lasx_xvreplgr2vr_w(qh[ib32+1]); + idx.vec[0] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[0], idx_shift), idx_mask); + idx.vec[1] = __lasx_xvand_v(__lasx_xvsll_w(idx.vec[1], idx_shift), idx_mask); + idx.vec[0] = __lasx_xvor_v(idx.vec[0], lasx_ext16_32(lasx_extracti128(idx_l, 0))); + idx.vec[1] = __lasx_xvor_v(idx.vec[1], lasx_ext16_32(lasx_extracti128(idx_l, 1))); + + // At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than _mm256_set_epi32. Strange. + //const __m256i q2_1 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[0], 4); + //const __m256i q2_2 = _mm256_i32gather_epi32((const int *)iq3s_grid, idx.vec[1], 4); + const __m256i q2_1 = lasx_set_w( + iq3s_grid[idx.index[7]], iq3s_grid[idx.index[6]], iq3s_grid[idx.index[5]], iq3s_grid[idx.index[4]], + iq3s_grid[idx.index[3]], iq3s_grid[idx.index[2]], iq3s_grid[idx.index[1]], iq3s_grid[idx.index[0]] + ); + const __m256i q2_2 = lasx_set_w( + iq3s_grid[idx.index[15]], iq3s_grid[idx.index[14]], iq3s_grid[idx.index[13]], iq3s_grid[idx.index[12]], + iq3s_grid[idx.index[11]], iq3s_grid[idx.index[10]], iq3s_grid[idx.index[ 9]], iq3s_grid[idx.index[ 8]] + ); + + __m256i aux256 = __lasx_xvreplgr2vr_w(signs[0] | (signs[1] << 16)); + aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2); + const __m256i s2_1 = __lasx_xvseq_b(aux256, mask2); + const __m256i q8s_1 = __lasx_xvsub_b(__lasx_xvxor_v(s2_1, q8_1), s2_1); + + aux256 = __lasx_xvreplgr2vr_w(signs[2] | (signs[3] << 16)); + aux256 = __lasx_xvand_v(lasx_shuffle_b(aux256,mask1), mask2); + const __m256i s2_2 = __lasx_xvseq_b(aux256, mask2); + const __m256i q8s_2 = __lasx_xvsub_b(__lasx_xvxor_v(s2_2, q8_2), s2_2); + + signs += 4; + + const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); + const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); + const uint16_t ls1 = x[i].scales[ib32/2] & 0xf; + const uint16_t ls2 = x[i].scales[ib32/2] >> 4; + const __m256i p1 = lasx_madd_h(dot1, __lasx_xvreplgr2vr_h(2*ls1+1)); + const __m256i p2 = lasx_madd_h(dot2, __lasx_xvreplgr2vr_h(2*ls2+1)); + sumi1 = __lasx_xvadd_w(sumi1, p1); + sumi2 = __lasx_xvadd_w(sumi2, p2); + } + + accumf = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accumf); + } + + *s = hsum_float_8(accumf); + #else float sumf = 0.f; @@ -11249,12 +13149,22 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * } -#ifdef __AVX2__ +#if defined(__AVX2__) static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { const __m256i ax = _mm256_sign_epi8(x, x); const __m256i sy = _mm256_sign_epi8(y, x); return _mm256_maddubs_epi16(ax, sy); } +#elif defined(__loongarch_asx) +static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { + const __m256i ax = __lasx_xvsigncov_b(x, x); + const __m256i sy = __lasx_xvsigncov_b(x, y); + __m256i tmp1, tmp2, tmp3; + tmp1 = __lasx_xvmulwev_h_bu_b(ax, sy); + tmp2 = __lasx_xvmulwod_h_bu_b(ax, sy); + tmp3 = __lasx_xvadd_h(tmp1, tmp2); + return __lasx_xvsat_h(tmp3, 15); +} #endif void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { @@ -11463,6 +13373,62 @@ void ggml_vec_dot_iq1_s_q8_K (int n, float * restrict s, size_t bs, const void vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + + __m256 accum = (__m256)__lasx_xvldi(0); + float accum1 = 0; + for (int i = 0; i < nb; ++i) { + + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint16_t * qh = x[i].qh; + + __m256i sumi = __lasx_xvldi(0); + int sumi1 = 0; + for (int ib = 0; ib < QK_K/32; ib += 2) { + __m256i q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[0] | ((qh[ib+0] << 8) & 0x700)], 0); + q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[1] | ((qh[ib+0] << 5) & 0x700)], 1); + q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[2] | ((qh[ib+0] << 2) & 0x700)], 2); + q1b_1 = __lasx_xvinsgr2vr_d(q1b_1, iq1s_grid[qs[3] | ((qh[ib+0] >> 1) & 0x700)], 3); + + __m256i q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[4] | ((qh[ib+1] << 8) & 0x700)], 0); + q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[5] | ((qh[ib+1] << 5) & 0x700)], 1); + q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[6] | ((qh[ib+1] << 2) & 0x700)], 2); + q1b_2 = __lasx_xvinsgr2vr_d(q1b_2, iq1s_grid[qs[7] | ((qh[ib+1] >> 1) & 0x700)], 3); + + qs += 8; + const __m256i q8b_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + const __m256i q8b_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; + + const __m256i dot1 = mul_add_epi8(q1b_1, q8b_1); + const __m256i dot2 = mul_add_epi8(q1b_2, q8b_2); + const int16_t ls1 = 2*((qh[ib+0] >> 12) & 7) + 1; + const int16_t ls2 = 2*((qh[ib+1] >> 12) & 7) + 1; + + __m256i tmp1, tmp5, tmp6; + tmp1 = __lasx_xvreplgr2vr_h(ls1); + tmp5 = __lasx_xvmulwev_w_h(dot1, tmp1); + tmp6 = __lasx_xvmulwod_w_h(dot1, tmp1); + const __m256i p1 = __lasx_xvadd_w(tmp5, tmp6); + + tmp1 = __lasx_xvreplgr2vr_h(ls2); + tmp5 = __lasx_xvmulwev_w_h(dot2, tmp1); + tmp6 = __lasx_xvmulwod_w_h(dot2, tmp1); + const __m256i p2 = __lasx_xvadd_w(tmp5, tmp6); + + sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p1, p2)); + sumi1 += (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]) * (qh[ib+0] & 0x8000 ? -1 : 1) * ls1 + + (y[i].bsums[2*ib+2] + y[i].bsums[2*ib+3]) * (qh[ib+1] & 0x8000 ? -1 : 1) * ls2; + } + + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), accum); + accum1 += d * sumi1; + } + + *s = hsum_float_8(accum) + IQ1S_DELTA * accum1; + #else float sumf = 0; @@ -11864,6 +13830,39 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined (__loongarch_asx) + + const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0); + const __m128i m4b = __lsx_vreplgr2vr_b(0x0f); + const __m256i mone = __lasx_xvreplgr2vr_h(1); + + __m256 accum1 = (__m256)__lasx_xvldi(0); + __m256 accum2 = (__m256)__lasx_xvldi(0); + for (int ib = 0; ib < nb; ib += 2) { + const __m128i q4bits_1 = __lsx_vld((const __m128i*)x[0].qs, 0); + const __m128i q4bits_2 = __lsx_vld((const __m128i*)x[1].qs, 0); + const __m256i q8b_1 = __lasx_xvld((const __m256i *)y[0].qs, 0); + const __m256i q8b_2 = __lasx_xvld((const __m256i *)y[1].qs, 0); + const __m256i q4b_1 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b)), + lsx_shuffle_b(values128, __lsx_vand_v(q4bits_1, m4b))); + const __m256i q4b_2 = lasx_insertf128(lsx_shuffle_b(values128, __lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b)), + lsx_shuffle_b(values128, __lsx_vand_v(q4bits_2, m4b))); + const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1); + const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2); + const __m256i p_1 = lasx_madd_h(p16_1, mone); + const __m256i p_2 = lasx_madd_h(p16_2, mone); + accum1 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[0].d)*GGML_FP16_TO_FP32(x[0].d)), + __lasx_xvffint_s_w(p_1), accum1); + accum2 = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(y[1].d)*GGML_FP16_TO_FP32(x[1].d)), + __lasx_xvffint_s_w(p_2), accum2); + + y += 2; + x += 2; + } + + *s = hsum_float_8(__lasx_xvfadd_s(accum1, accum2)); + #else float sumf = 0; for (int ib = 0; ib < nb; ++ib) { @@ -12074,6 +14073,80 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); *s = vec_extract(vsumf0, 0); + +#elif defined(__loongarch_asx) + + const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0); + const __m128i m4b = __lsx_vreplgr2vr_b(0x0f); + + __m256 accum = (__m256)__lasx_xvldi(0); + __m256i tmp1; + __m128i tmp0, tmp2, tmp3, tmp4, mask_8f, mask; + + mask_8f = __lsx_vreplgr2vr_b(0x8f); + for (int ibl = 0; ibl < nb; ++ibl) { + const uint8_t * qs = x[ibl].qs; + const int8_t * q8 = y[ibl].qs; + uint16_t sh = x[ibl].scales_h; + __m256i sumi1 = __lasx_xvldi(0); + __m256i sumi2 = __lasx_xvldi(0); + __m128i zero = __lsx_vldi(0); + for (int ib = 0; ib < QK_K/32; ib += 2) { + const __m128i q4bits_1 = __lsx_vld((const __m128i*)qs, 0); qs += 16; + const __m128i q4bits_2 = __lsx_vld((const __m128i*)qs, 0); qs += 16; + const __m256i q8b_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + const __m256i q8b_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; + tmp2 = __lsx_vand_v(__lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b), mask_8f); + tmp0 = __lsx_vori_b(tmp2, 0x10); + mask = __lsx_vsle_b(zero, tmp2); + tmp3 = __lsx_vand_v(tmp0, mask); + tmp3 = __lsx_vshuf_b(values128, zero, tmp3); + + tmp2 = __lsx_vand_v(__lsx_vand_v(q4bits_1, m4b), mask_8f); + tmp0 = __lsx_vori_b(tmp2, 0x10); + mask = __lsx_vsle_b(zero, tmp2); + tmp4 = __lsx_vand_v(tmp0, mask); + tmp4 = __lsx_vshuf_b(values128, zero, tmp4); + + const __m256i q4b_1 = lasx_insertf128(tmp3, tmp4); + + tmp2 = __lsx_vand_v(__lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b), mask_8f); + tmp0 = __lsx_vori_b(tmp2, 0x10); + mask = __lsx_vsle_b(zero, tmp2); + tmp3 = __lsx_vand_v(tmp0, mask); + tmp3 = __lsx_vshuf_b(values128, zero, tmp3); + + tmp2 = __lsx_vand_v(__lsx_vand_v(q4bits_2, m4b), mask_8f); + tmp0 = __lsx_vori_b(tmp2, 0x10); + mask = __lsx_vsle_b(zero, tmp2); + tmp4 = __lsx_vand_v(tmp0, mask); + tmp4 = __lsx_vshuf_b(values128, zero, tmp4); + + const __m256i q4b_2 = lasx_insertf128(tmp3, tmp4); + + const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1); + const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2); + const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32; + const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32; + sh >>= 4; + __m256i tmp5, tmp6; + tmp1 = __lasx_xvreplgr2vr_h(ls1); + tmp5 = __lasx_xvmulwev_w_h(p16_1, tmp1); + tmp6 = __lasx_xvmulwod_w_h(p16_1, tmp1); + const __m256i p_1 = __lasx_xvadd_w(tmp5, tmp6); + tmp1 = __lasx_xvreplgr2vr_h(ls2); + tmp5 = __lasx_xvmulwev_w_h(p16_2, tmp1); + tmp6 = __lasx_xvmulwod_w_h(p16_2, tmp1); + const __m256i p_2 = __lasx_xvadd_w(tmp5, tmp6); + sumi1 = __lasx_xvadd_w(p_1, sumi1); + sumi2 = __lasx_xvadd_w(p_2, sumi2); + } + accum = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(GGML_FP16_TO_FP32(x[ibl].d)*y[ibl].d), + __lasx_xvffint_s_w(__lasx_xvadd_w(sumi1, sumi2)), accum); + } + + *s = hsum_float_8(accum); + #else float sumf = 0; for (int ibl = 0; ibl < nb; ++ibl) { diff --git a/ggml.c b/ggml.c index 53da231ee..4bd911528 100644 --- a/ggml.c +++ b/ggml.c @@ -1523,6 +1523,195 @@ static inline void __sse_f16x4_store(ggml_fp16_t *x, __m128 y) { #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE +#elif defined(__loongarch_asx) + +#define GGML_SIMD + +// F32 LASX +#define GGML_F32_STEP 32 +#define GGML_F32_EPR 8 + +#define GGML_F32x8 __m256 +#define GGML_F32x8_ZERO (__m256)__lasx_xvldi(0) +#define GGML_F32x8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x)) +#define GGML_F32x8_LOAD(x) (__m256)__lasx_xvld((x), 0) +#define GGML_F32x8_STORE(x,y) __lasx_xvst((y), (x), 0) +#define GGML_F32x8_FMA(a, b, c) __lasx_xvfmadd_s(b, c, a) +#define GGML_F32x8_ADD __lasx_xvfadd_s +#define GGML_F32x8_MUL __lasx_xvfmul_s +#define GGML_F32x8_REDUCE(res, x) \ +do { \ + int offset = GGML_F32_ARR >> 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ + } \ + float *tmp_p = (float *)&x[0]; \ + res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7]; \ +} while (0) +// TODO: is this optimal ? + +#define GGML_F32_VEC GGML_F32x8 +#define GGML_F32_VEC_ZERO GGML_F32x8_ZERO +#define GGML_F32_VEC_SET1 GGML_F32x8_SET1 +#define GGML_F32_VEC_LOAD GGML_F32x8_LOAD +#define GGML_F32_VEC_STORE GGML_F32x8_STORE +#define GGML_F32_VEC_FMA GGML_F32x8_FMA +#define GGML_F32_VEC_ADD GGML_F32x8_ADD +#define GGML_F32_VEC_MUL GGML_F32x8_MUL +#define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE + +// F16 LASX + +#define GGML_F16_STEP 32 +#define GGML_F16_EPR 8 + +// F16 arithmetic is not supported by AVX, so we use F32 instead + +#define GGML_F32Cx8 __m256 +#define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0) +#define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplgr2vr_w((x)) + +static inline __m256 __lasx_f32cx8_load(ggml_fp16_t *x) { + float tmp[8]; + + for (int i = 0; i < 8; i++) { + tmp[i] = GGML_FP16_TO_FP32(x[i]); + } + + return (__m256)__lasx_xvld(tmp, 0); +} +static inline void __lasx_f32cx8_store(ggml_fp16_t *x, __m256 y) { + float arr[8]; + + __lasx_xvst(y, arr, 0); + + for (int i = 0; i < 8; i++) + x[i] = GGML_FP32_TO_FP16(arr[i]); +} +#define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x) +#define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y) + +#define GGML_F32Cx8_FMA GGML_F32x8_FMA +#define GGML_F32Cx8_ADD __lasx_xvfadd_s +#define GGML_F32Cx8_MUL __lasx_xvfmul_s +#define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE + +#define GGML_F16_VEC GGML_F32Cx8 +#define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO +#define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1 +#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p) +#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i]) +#define GGML_F16_VEC_FMA GGML_F32Cx8_FMA +#define GGML_F16_VEC_ADD GGML_F32Cx8_ADD +#define GGML_F16_VEC_MUL GGML_F32Cx8_MUL +#define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE + +#elif defined(__loongarch_sx) + +#define GGML_SIMD + +// F32 LSX + +#define GGML_F32_STEP 32 +#define GGML_F32_EPR 4 + +#define GGML_F32x4 __m128 +#define GGML_F32x4_ZERO __lsx_vldi(0) +#define GGML_F32x4_SET1(x) __lsx_vinsgr2vr_w(__lsx_vldi(0),(x), 0) +#define GGML_F32x4_LOAD(x) __lsx_vld((x), 0) +#define GGML_F32x4_STORE((x),(y)) __lsx_vst((y), (x), 0) +#define GGML_F32x4_FMA(a, b, c) __lsx_vfmadd_s(b, c, a) +#define GGML_F32x4_ADD __lsx_vfadd_s +#define GGML_F32x4_MUL __lsx_vfmul_s +#define GGML_F32x4_REDUCE(res, x) \ +{ \ + int offset = GGML_F32_ARR >> 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ + } \ + offset >>= 1; \ + for (int i = 0; i < offset; ++i) { \ + x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ + } \ + __m128i tmp = __lsx_vsrli_d((__m128i)x[0], 32); \ + tmp = (__m128i)__lsx_vfadd_s((__m128)tmp, x[0]); \ + tmp = __lsx_vpickev_w(__lsx_vldi(0), tmp); \ + const __m128 t0 = __lsx_vshuf4i_w(tmp, 0x88); \ + tmp = __lsx_vsrli_d((__m128i)t0, 32); \ + tmp = (__m128i)__lsx_vfadd_s((__m128)tmp, t0); \ + tmp = __lsx_vpickev_w(__lsx_vldi(0), tmp); \ + res = (ggml_float) __lsx_vpickve2gr_w(__lsx_vshuf4i_w(tmp, 0x88), 0); \ +} + +#define GGML_F32_VEC GGML_F32x4 +#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO +#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 +#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD +#define GGML_F32_VEC_STORE GGML_F32x4_STORE +#define GGML_F32_VEC_FMA GGML_F32x4_FMA +#define GGML_F32_VEC_ADD GGML_F32x4_ADD +#define GGML_F32_VEC_MUL GGML_F32x4_MUL +#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE + +// F16 LSX + +#define GGML_F16_STEP 32 +#define GGML_F16_EPR 4 + +static inline __m128 __lsx_f16x4_load(ggml_fp16_t *x) { + float tmp[4]; + + tmp[0] = GGML_FP16_TO_FP32(x[0]); + tmp[1] = GGML_FP16_TO_FP32(x[1]); + tmp[2] = GGML_FP16_TO_FP32(x[2]); + tmp[3] = GGML_FP16_TO_FP32(x[3]); + + return __lsx_vld(tmp, 0); +} + +static inline void __lsx_f16x4_store(ggml_fp16_t *x, __m128 y) { + float arr[4]; + + __lsx_vst(y, arr, 0); + + x[0] = GGML_FP32_TO_FP16(arr[0]); + x[1] = GGML_FP32_TO_FP16(arr[1]); + x[2] = GGML_FP32_TO_FP16(arr[2]); + x[3] = GGML_FP32_TO_FP16(arr[3]); +} + +#define GGML_F32Cx4 __m128 +#define GGML_F32Cx4_ZERO __lsx_vldi(0) +#define GGML_F32Cx4_SET1(x) __lsx_vinsgr2vr_w(__lsx_vldi(0),(x), 0) +#define GGML_F32Cx4_LOAD(x) __lsx_f16x4_load(x) +#define GGML_F32Cx4_STORE(x, y) __lsx_f16x4_store(x, y) +#define GGML_F32Cx4_FMA GGML_F32x4_FMA +#define GGML_F32Cx4_ADD __lsx_vfadd_s +#define GGML_F32Cx4_MUL __lsx_vfmul_s +#define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE + +#define GGML_F16_VEC GGML_F32Cx4 +#define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO +#define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 +#define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p) +#define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i]) +#define GGML_F16_VEC_FMA GGML_F32Cx4_FMA +#define GGML_F16_VEC_ADD GGML_F32Cx4_ADD +#define GGML_F16_VEC_MUL GGML_F32Cx4_MUL +#define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE + #endif // GGML_F32_ARR / GGML_F16_ARR From 213e90ed73f8ac3cd3026dc3f086beae0d414f96 Mon Sep 17 00:00:00 2001 From: Herman Semenov Date: Mon, 20 May 2024 07:33:21 +0000 Subject: [PATCH 085/137] ggml-opencl, llama: using reserve() if count already known (#7272) --- ggml-opencl.cpp | 7 +++++-- llama.cpp | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ggml-opencl.cpp b/ggml-opencl.cpp index 880a14958..922f24837 100644 --- a/ggml-opencl.cpp +++ b/ggml-opencl.cpp @@ -1,4 +1,4 @@ -#include "ggml.h" +#include "ggml.h" #include "ggml-opencl.h" #include "ggml-backend-impl.h" @@ -1835,7 +1835,10 @@ static void ggml_cl_mul_mat_q_f32(const ggml_tensor * src0, const ggml_tensor * CL_CHECK(clEnqueueNDRangeKernel(queue, *to_fp32_cl, 1, &offset, &global, local > 0 ? &local : NULL, events.size(), !events.empty() ? events.data() : NULL, NULL)); } - for (int64_t i12 = i02 * r2, e12 = i12 + r2; i12 < e12; i12++) { + int64_t i12 = i02 * r2; + int64_t e12 = i12 + r2; + events.reserve(e12 - i12); + for (; i12 < e12; i12++) { if (mul_mat_vec) { // specialized dequantize_mul_mat_vec kernel // copy src1 to device events.emplace_back(); diff --git a/llama.cpp b/llama.cpp index ca3e9fcc0..2025e4558 100644 --- a/llama.cpp +++ b/llama.cpp @@ -16162,6 +16162,7 @@ static bool llama_control_vector_init(struct llama_control_vector & cvec, const } // make tensors + cvec.tensors.reserve(model.hparams.n_layer); cvec.tensors.push_back(nullptr); // there's never a tensor for layer 0 for (size_t il = 1; il < model.hparams.n_layer; il++) { struct ggml_context * ctx = ctx_map.at(model.buft_layer[il].buft); @@ -16170,6 +16171,8 @@ static bool llama_control_vector_init(struct llama_control_vector & cvec, const } // allocate tensors / buffers and zero + cvec.ctxs.reserve(ctx_map.size()); + cvec.bufs.reserve(ctx_map.size()); for (auto it : ctx_map) { ggml_backend_buffer_type_t buft = it.first; ggml_context * ctx = it.second; From 26cd4237bc499f7144d76f440aa775d749b170bb Mon Sep 17 00:00:00 2001 From: Bingan <70050083+binganao@users.noreply.github.com> Date: Mon, 20 May 2024 17:55:34 +0800 Subject: [PATCH 086/137] Update README.md (#7410) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4abfd6d7e..47d41ebfc 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,7 @@ cd llama.cpp ### Build -In order to build llama.cpp you have three different options. +In order to build llama.cpp you have four different options. - Using `make`: - On Linux or MacOS: From 6bf9b66fa3f263ca2175dcb5f6d0a658581e1dfb Mon Sep 17 00:00:00 2001 From: AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> Date: Mon, 20 May 2024 12:08:23 +0100 Subject: [PATCH 087/137] [SYCL] Update SYCL upscale operation (#7321) * Update SYCL upscale operation * Formatting * Remove messages --- ggml-sycl.cpp | 65 +++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index 19d22d637..eac8f5579 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -3847,21 +3847,27 @@ static void concat_f32(const float *x,const float *y, float *dst, const int ne } } -static void upscale_f32(const float *x, float *dst, const int ne00, const int nb02, const int scale_factor, - const sycl::nd_item<3> &item_ct1) { - int ne0 = ne00 * scale_factor; - int nidx = item_ct1.get_local_id(2) + - item_ct1.get_group(2) * item_ct1.get_local_range(2); - if (nidx >= ne0) { +static void upscale_f32(const float *x, float *dst, const int nb00, const int nb01, + const int nb02, const int nb03, const int ne10, const int ne11, + const int ne12, const int ne13, const float sf0, const float sf1, + const float sf2, const float sf3, const sycl::nd_item<1> &item_ct1) { + int index = item_ct1.get_local_id(0) + + item_ct1.get_group(0) * item_ct1.get_local_range(0); + if (index >= ne10 * ne11 * ne12 * ne13) { return; } // operation - int i00 = nidx / scale_factor; - int i01 = item_ct1.get_group(1) / scale_factor; - int offset_src = i00 + i01 * ne00 + item_ct1.get_group(0) * nb02; - int offset_dst = nidx + item_ct1.get_group(1) * ne0 + - item_ct1.get_group(0) * ne0 * item_ct1.get_group_range(1); - dst[offset_dst] = x[offset_src]; + int i10 = index % ne10; + int i11 = (index / ne10) % ne11; + int i12 = (index / (ne10 * ne11)) % ne12; + int i13 = (index / (ne10 * ne11 * ne12)) % ne13; + + int i00 = i10 / sf0; + int i01 = i11 / sf1; + int i02 = i12 / sf2; + int i03 = i13 / sf3; + + dst[index] = *(float *)((char *)x + i03 * nb03 + i02 * nb02 + i01 * nb01 + i00 * nb00); } static void pad_f32(const float *x, float *dst, const int ne0, const int ne00, const int ne01, const int ne02, @@ -10085,18 +10091,17 @@ static void concat_f32_sycl(const float *x, const float *y, float *dst, }); } -static void upscale_f32_sycl(const float *x, float *dst, const int ne00, - const int ne01, const int ne02, - const int scale_factor, dpct::queue_ptr stream) { - int ne0 = (ne00 * scale_factor); - int num_blocks = (ne0 + SYCL_UPSCALE_BLOCK_SIZE - 1) / SYCL_UPSCALE_BLOCK_SIZE; - sycl::range<3> gridDim(ne02, (ne01 * scale_factor), num_blocks); +static void upscale_f32_sycl(const float *x, float *dst, const int nb00, const int nb01, + const int nb02, const int nb03, const int ne10, const int ne11, + const int ne12, const int ne13, const float sf0, const float sf1, + const float sf2, const float sf3, dpct::queue_ptr stream) { + int dst_size = ne10 * ne11 * ne12 * ne13; + int num_blocks = (dst_size + SYCL_UPSCALE_BLOCK_SIZE - 1) / SYCL_UPSCALE_BLOCK_SIZE; + sycl::range<1> gridDim(num_blocks * SYCL_UPSCALE_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(gridDim * - sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_UPSCALE_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { - upscale_f32(x, dst, ne00, ne00 * ne01, scale_factor, item_ct1); + sycl::nd_range<1>(gridDim, sycl::range<1>(SYCL_UPSCALE_BLOCK_SIZE)), + [=](sycl::nd_item<1> item_ct1) { + upscale_f32(x, dst, nb00, nb01, nb02, nb03, ne10, ne11, ne12, ne13, sf0, sf1, sf2, sf3, item_ct1); }); } @@ -13985,15 +13990,15 @@ inline void ggml_sycl_op_upscale(const ggml_tensor *src0, GGML_ASSERT(src0->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); - GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors -#pragma message("TODO: generalize upscale operator") -#pragma message(" https://github.com/ggerganov/ggml/pull/814") - GGML_ASSERT(false && "TODO: generalize upscale operator"); + const float sf0 = (float)dst->ne[0]/src0->ne[0]; + const float sf1 = (float)dst->ne[1]/src0->ne[1]; + const float sf2 = (float)dst->ne[2]/src0->ne[2]; + const float sf3 = (float)dst->ne[3]/src0->ne[3]; - const int scale_factor = dst->op_params[0]; - - upscale_f32_sycl(src0_dd, dst_dd, src0->ne[0], src0->ne[1], src0->ne[2], scale_factor, main_stream); + upscale_f32_sycl(src0_dd, dst_dd, src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], + dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], sf0, sf1, sf2, sf3, + main_stream); (void) src1; (void) dst; From 3bc10cb485dd7efa4da6c64e73ad0c9e2bfe0821 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 20 May 2024 15:10:03 +0300 Subject: [PATCH 088/137] server : fix temperature + disable some tests (#7409) * server : fix temperature * server : disable tests relying on parallel determinism * ci : change server Debug -> RelWithDebInfo --- .github/workflows/server.yml | 7 +------ examples/server/tests/features/results.feature | 17 ++++++++--------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 217af67cf..0789efd18 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -33,13 +33,10 @@ jobs: strategy: matrix: sanitizer: [ADDRESS, THREAD, UNDEFINED] - build_type: [Debug] + build_type: [RelWithDebInfo] include: - build_type: Release sanitizer: "" - - build_type: Debug - sanitizer: THREAD - disabled_on_pr: true fail-fast: false # While -DLLAMA_SANITIZE_THREAD=ON is broken steps: @@ -103,10 +100,8 @@ jobs: -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON ; cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target server - - name: Tests id: server_integration_tests - if: ${{ !matrix.disabled_on_pr || !github.event.pull_request }} run: | cd examples/server/tests PORT=8888 ./tests.sh diff --git a/examples/server/tests/features/results.feature b/examples/server/tests/features/results.feature index 4ab8ad20c..e8e1b5414 100644 --- a/examples/server/tests/features/results.feature +++ b/examples/server/tests/features/results.feature @@ -13,7 +13,7 @@ Feature: Results Scenario Outline: consistent results with same seed Given slots - And 0.0 temperature + And 1.0 temperature Then the server is starting Then the server is healthy @@ -27,7 +27,8 @@ Feature: Results Examples: | n_slots | | 1 | - | 2 | + # FIXME: unified KV cache nondeterminism + # | 2 | Scenario Outline: different results with different seed Given slots @@ -73,14 +74,13 @@ Feature: Results Examples: | n_parallel | temp | | 1 | 0.0 | - | 2 | 0.0 | - | 4 | 0.0 | | 1 | 1.0 | - # FIXME: These tests fail on master. - # Problems: unified KV cache (except for CPU backend with LLAMA_NO_LLAMAFILE=1), SIMD nondeterminism. + # FIXME: unified KV cache nondeterminism # See https://github.com/ggerganov/whisper.cpp/issues/1941#issuecomment-1986923227 # and https://github.com/ggerganov/llama.cpp/pull/6122#discussion_r1531405574 # and https://github.com/ggerganov/llama.cpp/pull/7347 . + # | 2 | 0.0 | + # | 4 | 0.0 | # | 2 | 1.0 | # | 4 | 1.0 | @@ -108,12 +108,11 @@ Feature: Results Examples: | n_slots | n_kv | n_predict | n_parallel | | 4 | 1024 | 1 | 1 | - | 4 | 1024 | 1 | 4 | - # FIXME: These tests fail on master. - # Problems: unified KV cache (except for CPU backend with LLAMA_NO_LLAMAFILE=1), SIMD nondeterminism. + # FIXME: unified KV cache nondeterminism # See https://github.com/ggerganov/whisper.cpp/issues/1941#issuecomment-1986923227 # and https://github.com/ggerganov/llama.cpp/pull/6122#discussion_r1531405574 # and https://github.com/ggerganov/llama.cpp/pull/7347 . + # | 4 | 1024 | 1 | 4 | # | 4 | 1024 | 100 | 1 | # This test still fails even the above patches; the first token probabilities are already different. # | 4 | 1024 | 100 | 4 | From db10f01310beea8a1ef7798651b9d692fd1149d0 Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Mon, 20 May 2024 16:36:55 +0300 Subject: [PATCH 089/137] rpc : track allocated buffers (#7411) * rpc : track allocated buffers ref: #7407 * rpc : pack rpc_tensor tightly --- ggml-rpc.cpp | 229 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 176 insertions(+), 53 deletions(-) diff --git a/ggml-rpc.cpp b/ggml-rpc.cpp index 4a9bfa52d..cc1d3ace1 100644 --- a/ggml-rpc.cpp +++ b/ggml-rpc.cpp @@ -56,6 +56,7 @@ struct socket_t { }; // ggml_tensor is serialized into rpc_tensor +#pragma pack(push, 1) struct rpc_tensor { uint64_t id; uint32_t type; @@ -71,6 +72,7 @@ struct rpc_tensor { uint64_t data; char name[GGML_MAX_NAME]; }; +#pragma pack(pop) // RPC commands enum rpc_cmd { @@ -340,23 +342,6 @@ static rpc_tensor serialize_tensor(const ggml_tensor * tensor) { return result; } -static ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) { - ggml_tensor * result = ggml_new_tensor_4d(ctx, (ggml_type) tensor->type, - tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); - for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) { - result->nb[i] = tensor->nb[i]; - } - result->buffer = reinterpret_cast(tensor->buffer); - result->op = (ggml_op) tensor->op; - for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) { - result->op_params[i] = tensor->op_params[i]; - } - result->flags = tensor->flags; - result->data = reinterpret_cast(tensor->data); - ggml_set_name(result, tensor->name); - return result; -} - GGML_CALL static void ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { UNUSED(buffer); if (ggml_is_quantized(tensor->type)) { @@ -465,13 +450,15 @@ GGML_CALL static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer memcpy(&remote_ptr, output.data(), sizeof(remote_ptr)); size_t remote_size; memcpy(&remote_size, output.data() + sizeof(uint64_t), sizeof(remote_size)); - - ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft, - ggml_backend_rpc_buffer_interface, - new ggml_backend_rpc_buffer_context{buft_ctx->sock, {}, remote_ptr, "RPC"}, - remote_size); - - return buffer; + if (remote_ptr != 0) { + ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft, + ggml_backend_rpc_buffer_interface, + new ggml_backend_rpc_buffer_context{buft_ctx->sock, {}, remote_ptr, "RPC"}, + remote_size); + return buffer; + } else { + return nullptr; + } } static size_t get_alignment(const std::shared_ptr & sock) { @@ -658,7 +645,7 @@ GGML_CALL ggml_backend_t ggml_backend_rpc_init(const char * endpoint) { } } #endif - GGML_PRINT_DEBUG("Connecting to %s\n", endpoint); + fprintf(stderr, "Connecting to %s\n", endpoint); std::string host; int port; if (!parse_endpoint(endpoint, host, port)) { @@ -731,22 +718,61 @@ GGML_API GGML_CALL void ggml_backend_rpc_get_device_memory(const char * endpoint // RPC server-side implementation -static void rpc_alloc_buffer(ggml_backend_t backend, const std::vector & input, std::vector & output) { +class rpc_server { +public: + rpc_server(ggml_backend_t backend) : backend(backend) {} + ~rpc_server(); + + bool alloc_buffer(const std::vector & input, std::vector & output); + void get_alignment(std::vector & output); + void get_max_size(std::vector & output); + bool buffer_get_base(const std::vector & input, std::vector & output); + bool free_buffer(const std::vector & input); + bool buffer_clear(const std::vector & input); + bool set_tensor(const std::vector & input); + bool get_tensor(const std::vector & input, std::vector & output); + bool copy_tensor(const std::vector & input, std::vector & output); + bool graph_compute(const std::vector & input, std::vector & output); + +private: + ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor); + ggml_tensor * create_node(uint64_t id, + struct ggml_context * ctx, + const std::unordered_map & tensor_ptrs, + std::unordered_map & tensor_map); + + + ggml_backend_t backend; + std::unordered_set buffers; +}; + +bool rpc_server::alloc_buffer(const std::vector & input, std::vector & output) { // input serialization format: | size (8 bytes) | + if (input.size() != sizeof(uint64_t)) { + return false; + } uint64_t size; memcpy(&size, input.data(), sizeof(size)); ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, size); - uint64_t remote_ptr = reinterpret_cast(buffer); - uint64_t remote_size = buffer->size; - GGML_PRINT_DEBUG("[%s] size: %" PRIu64 " -> remote_ptr: %" PRIx64 ", remote_size: %" PRIu64 "\n", __func__, size, remote_ptr, remote_size); + uint64_t remote_ptr = 0; + uint64_t remote_size = 0; + if (buffer != nullptr) { + remote_ptr = reinterpret_cast(buffer); + remote_size = buffer->size; + GGML_PRINT_DEBUG("[%s] size: %" PRIu64 " -> remote_ptr: %" PRIx64 ", remote_size: %" PRIu64 "\n", __func__, size, remote_ptr, remote_size); + buffers.insert(buffer); + } else { + GGML_PRINT_DEBUG("[%s] size: %" PRIu64 " -> failed\n", __func__, size); + } // output serialization format: | remote_ptr (8 bytes) | remote_size (8 bytes) | output.resize(2*sizeof(uint64_t), 0); memcpy(output.data(), &remote_ptr, sizeof(remote_ptr)); memcpy(output.data() + sizeof(uint64_t), &remote_size, sizeof(remote_size)); + return true; } -static void rpc_get_alignment(ggml_backend_t backend, std::vector & output) { +void rpc_server::get_alignment(std::vector & output) { ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); size_t alignment = ggml_backend_buft_get_alignment(buft); GGML_PRINT_DEBUG("[%s] alignment: %lu\n", __func__, alignment); @@ -755,7 +781,7 @@ static void rpc_get_alignment(ggml_backend_t backend, std::vector & out memcpy(output.data(), &alignment, sizeof(alignment)); } -static void rpc_get_max_size(ggml_backend_t backend, std::vector & output) { +void rpc_server::get_max_size(std::vector & output) { ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); size_t max_size = ggml_backend_buft_get_max_size(buft); GGML_PRINT_DEBUG("[%s] max_size: %lu\n", __func__, max_size); @@ -764,41 +790,90 @@ static void rpc_get_max_size(ggml_backend_t backend, std::vector & outp memcpy(output.data(), &max_size, sizeof(max_size)); } -static void rpc_buffer_get_base(const std::vector & input, std::vector & output) { +bool rpc_server::buffer_get_base(const std::vector & input, std::vector & output) { // input serialization format: | remote_ptr (8 bytes) | + if (input.size() != sizeof(uint64_t)) { + return false; + } uint64_t remote_ptr; memcpy(&remote_ptr, input.data(), sizeof(remote_ptr)); GGML_PRINT_DEBUG("[%s] remote_ptr: %" PRIx64 "\n", __func__, remote_ptr); ggml_backend_buffer_t buffer = reinterpret_cast(remote_ptr); + if (buffers.find(buffer) == buffers.end()) { + GGML_PRINT_DEBUG("[%s] buffer not found\n", __func__); + return false; + } void * base = ggml_backend_buffer_get_base(buffer); // output serialization format: | base_ptr (8 bytes) | uint64_t base_ptr = reinterpret_cast(base); output.resize(sizeof(uint64_t), 0); memcpy(output.data(), &base_ptr, sizeof(base_ptr)); + return true; } -static void rpc_free_buffer(const std::vector & input) { +bool rpc_server::free_buffer(const std::vector & input) { // input serialization format: | remote_ptr (8 bytes) | + if (input.size() != sizeof(uint64_t)) { + return false; + } uint64_t remote_ptr; memcpy(&remote_ptr, input.data(), sizeof(remote_ptr)); GGML_PRINT_DEBUG("[%s] remote_ptr: %" PRIx64 "\n", __func__, remote_ptr); ggml_backend_buffer_t buffer = reinterpret_cast(remote_ptr); + if (buffers.find(buffer) == buffers.end()) { + GGML_PRINT_DEBUG("[%s] buffer not found\n", __func__); + return false; + } ggml_backend_buffer_free(buffer); + buffers.erase(buffer); + return true; } -static void rpc_buffer_clear(const std::vector & input) { +bool rpc_server::buffer_clear(const std::vector & input) { // input serialization format: | remote_ptr (8 bytes) | value (1 byte) | + if (input.size() != sizeof(uint64_t) + sizeof(uint8_t)) { + return false; + } uint64_t remote_ptr; memcpy(&remote_ptr, input.data(), sizeof(remote_ptr)); uint8_t value; memcpy(&value, input.data() + sizeof(uint64_t), sizeof(value)); GGML_PRINT_DEBUG("[%s] remote_ptr: %" PRIx64 ", value: %u\n", __func__, remote_ptr, value); ggml_backend_buffer_t buffer = reinterpret_cast(remote_ptr); + if (buffers.find(buffer) == buffers.end()) { + GGML_PRINT_DEBUG("[%s] buffer not found\n", __func__); + return false; + } ggml_backend_buffer_clear(buffer, value); + return true; } -static void rpc_set_tensor(const std::vector & input) { +ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) { + ggml_tensor * result = ggml_new_tensor_4d(ctx, (ggml_type) tensor->type, + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3]); + for (uint32_t i = 0; i < GGML_MAX_DIMS; i++) { + result->nb[i] = tensor->nb[i]; + } + result->buffer = reinterpret_cast(tensor->buffer); + if (result->buffer && buffers.find(result->buffer) == buffers.end()) { + return nullptr; + } + result->op = (ggml_op) tensor->op; + for (uint32_t i = 0; i < GGML_MAX_OP_PARAMS / sizeof(int32_t); i++) { + result->op_params[i] = tensor->op_params[i]; + } + result->flags = tensor->flags; + result->data = reinterpret_cast(tensor->data); + ggml_set_name(result, tensor->name); + return result; +} + + +bool rpc_server::set_tensor(const std::vector & input) { // serialization format: | rpc_tensor | offset (8 bytes) | data (size bytes) | + if (input.size() < sizeof(rpc_tensor) + sizeof(uint64_t)) { + return false; + } const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); uint64_t offset; memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset)); @@ -811,14 +886,23 @@ static void rpc_set_tensor(const std::vector & input) { }; struct ggml_context * ctx = ggml_init(params); ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor); + if (tensor == nullptr) { + GGML_PRINT_DEBUG("[%s] error deserializing tensor\n", __func__); + ggml_free(ctx); + return false; + } GGML_PRINT_DEBUG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu\n", __func__, (void*)tensor->buffer, tensor->data, offset, size); const void * data = input.data() + sizeof(rpc_tensor) + sizeof(offset); ggml_backend_tensor_set(tensor, data, offset, size); ggml_free(ctx); + return true; } -static void rpc_get_tensor(const std::vector & input, std::vector & output) { +bool rpc_server::get_tensor(const std::vector & input, std::vector & output) { // serialization format: | rpc_tensor | offset (8 bytes) | size (8 bytes) | + if (input.size() != sizeof(rpc_tensor) + 2*sizeof(uint64_t)) { + return false; + } const rpc_tensor * in_tensor = (const rpc_tensor *)input.data(); uint64_t offset; memcpy(&offset, input.data() + sizeof(rpc_tensor), sizeof(offset)); @@ -832,15 +916,24 @@ static void rpc_get_tensor(const std::vector & input, std::vectorbuffer, tensor->data, offset, size); // output serialization format: | data (size bytes) | output.resize(size, 0); ggml_backend_tensor_get(tensor, output.data(), offset, size); ggml_free(ctx); + return true; } -static void rpc_copy_tensor(const std::vector & input, std::vector & output) { +bool rpc_server::copy_tensor(const std::vector & input, std::vector & output) { // serialization format: | rpc_tensor src | rpc_tensor dst | + if (input.size() != 2*sizeof(rpc_tensor)) { + return false; + } const rpc_tensor * rpc_src = (const rpc_tensor *)input.data(); const rpc_tensor * rpc_dst = (const rpc_tensor *)(input.data() + sizeof(rpc_src)); @@ -852,18 +945,24 @@ static void rpc_copy_tensor(const std::vector & input, std::vectorbuffer: %p, dst->buffer: %p\n", __func__, (void*)src->buffer, (void*)dst->buffer); bool result = ggml_backend_buffer_copy_tensor(src, dst); // output serialization format: | result (1 byte) | output.resize(1, 0); output[0] = result; ggml_free(ctx); + return true; } -static struct ggml_tensor * create_node(uint64_t id, - struct ggml_context * ctx, - const std::unordered_map & tensor_ptrs, - std::unordered_map & tensor_map) { +ggml_tensor * rpc_server::create_node(uint64_t id, + struct ggml_context * ctx, + const std::unordered_map & tensor_ptrs, + std::unordered_map & tensor_map) { if (id == 0) { return nullptr; } @@ -872,6 +971,9 @@ static struct ggml_tensor * create_node(uint64_t id, } const rpc_tensor * tensor = tensor_ptrs.at(id); struct ggml_tensor * result = deserialize_tensor(ctx, tensor); + if (result == nullptr) { + return nullptr; + } tensor_map[id] = result; for (int i = 0; i < GGML_MAX_SRC; i++) { result->src[i] = create_node(tensor->src[i], ctx, tensor_ptrs, tensor_map); @@ -881,14 +983,23 @@ static struct ggml_tensor * create_node(uint64_t id, return result; } -static void rpc_graph_compute(ggml_backend_t backend, const std::vector & input, std::vector & output) { +bool rpc_server::graph_compute(const std::vector & input, std::vector & output) { // serialization format: // | n_nodes (4 bytes) | nodes (n_nodes * sizeof(uint64_t) | n_tensors (4 bytes) | tensors (n_tensors * sizeof(rpc_tensor)) | + if (input.size() < sizeof(uint32_t)) { + return false; + } uint32_t n_nodes; memcpy(&n_nodes, input.data(), sizeof(n_nodes)); + if (input.size() < sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t)) { + return false; + } const uint64_t * nodes = (const uint64_t *)(input.data() + sizeof(n_nodes)); uint32_t n_tensors; memcpy(&n_tensors, input.data() + sizeof(n_nodes) + n_nodes*sizeof(uint64_t), sizeof(n_tensors)); + if (input.size() < sizeof(uint32_t) + n_nodes*sizeof(uint64_t) + sizeof(uint32_t) + n_tensors*sizeof(rpc_tensor)) { + return false; + } const rpc_tensor * tensors = (const rpc_tensor *)(input.data() + sizeof(n_nodes) + n_nodes*sizeof(uint64_t) + sizeof(n_tensors)); GGML_PRINT_DEBUG("[%s] n_nodes: %u, n_tensors: %u\n", __func__, n_nodes, n_tensors); @@ -914,9 +1025,17 @@ static void rpc_graph_compute(ggml_backend_t backend, const std::vector output.resize(1, 0); output[0] = status; ggml_free(ctx); + return true; +} + +rpc_server::~rpc_server() { + for (auto buffer : buffers) { + ggml_backend_buffer_free(buffer); + } } static void rpc_serve_client(ggml_backend_t backend, sockfd_t sockfd, size_t free_mem, size_t total_mem) { + rpc_server server(backend); while (true) { uint8_t cmd; if (!recv_data(sockfd, &cmd, 1)) { @@ -932,45 +1051,46 @@ static void rpc_serve_client(ggml_backend_t backend, sockfd_t sockfd, size_t fre if (!recv_data(sockfd, input.data(), input_size)) { break; } + bool ok = true; switch (cmd) { case ALLOC_BUFFER: { - rpc_alloc_buffer(backend, input, output); + ok = server.alloc_buffer(input, output); break; } case GET_ALIGNMENT: { - rpc_get_alignment(backend, output); + server.get_alignment(output); break; } case GET_MAX_SIZE: { - rpc_get_max_size(backend, output); + server.get_max_size(output); break; } case BUFFER_GET_BASE: { - rpc_buffer_get_base(input, output); + ok = server.buffer_get_base(input, output); break; } case FREE_BUFFER: { - rpc_free_buffer(input); + ok = server.free_buffer(input); break; } case BUFFER_CLEAR: { - rpc_buffer_clear(input); + ok = server.buffer_clear(input); break; } case SET_TENSOR: { - rpc_set_tensor(input); + ok = server.set_tensor(input); break; } case GET_TENSOR: { - rpc_get_tensor(input, output); + ok = server.get_tensor(input, output); break; } case COPY_TENSOR: { - rpc_copy_tensor(input, output); + ok = server.copy_tensor(input, output); break; } case GRAPH_COMPUTE: { - rpc_graph_compute(backend, input, output); + ok = server.graph_compute(input, output); break; } case GET_DEVICE_MEMORY: { @@ -982,9 +1102,12 @@ static void rpc_serve_client(ggml_backend_t backend, sockfd_t sockfd, size_t fre } default: { fprintf(stderr, "Unknown command: %d\n", cmd); - return; + ok = false; } } + if (!ok) { + break; + } uint64_t output_size = output.size(); if (!send_data(sockfd, &output_size, sizeof(output_size))) { break; From 20385cebcc4bb3f6dd10f989573c11864d70d53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Mon, 20 May 2024 18:15:38 +0200 Subject: [PATCH 090/137] perplexity: update README FP16 results [no ci] (#7413) --- examples/perplexity/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/perplexity/README.md b/examples/perplexity/README.md index c2a3c5ce9..33a46d1a2 100644 --- a/examples/perplexity/README.md +++ b/examples/perplexity/README.md @@ -42,10 +42,13 @@ In addition to the KL divergence the following statistics are calculated with `- Results were generated using the CUDA backend and are sorted by Kullback-Leibler divergence relative to FP16. The "WT" importance matrices were created using varying numbers of Wikitext tokens and can be found [here](https://huggingface.co/JohannesGaessler/llama.cpp_importance_matrices/blob/main/imatrix-llama_3-8b-f16-2.7m_tokens.dat). +Note: the FP16 logits used for the calculation of all metrics other than perplexity are stored in a binary file between runs. +In order to save space this file does **not** contain the exact same FP32 logits but instead casts them to 16 bit unsigned integers (with some scaling). +So the "f16" results are to be understood as the difference resulting only from this downcast. | Quantization | imatrix | Model size [GiB] | PPL | ΔPPL | KLD | Mean Δp | RMS Δp | |--------------|---------|------------------|------------------------|------------------------|-----------------------|-------------------|------------------| -| f16 | None | 14.97 | 6.233160 ± 0.037828 | - | - | - | - | +| f16 | None | 14.97 | 6.233160 ± 0.037828 | 0.001524 ± 0.000755 | 0.000551 ± 0.000002 | 0.001 ± 0.002 % | 0.787 ± 0.004 % | | q8_0 | None | 7.96 | 6.234284 ± 0.037878 | 0.002650 ± 0.001006 | 0.001355 ± 0.000006 | -0.019 ± 0.003 % | 1.198 ± 0.007 % | | q6_K | None | 6.14 | 6.253382 ± 0.038078 | 0.021748 ± 0.001852 | 0.005452 ± 0.000035 | -0.007 ± 0.006 % | 2.295 ± 0.019 % | | q5_K_M | None | 5.33 | 6.288607 ± 0.038338 | 0.056974 ± 0.002598 | 0.010762 ± 0.000079 | -0.114 ± 0.008 % | 3.160 ± 0.031 % | From fabf30b4c4fca32e116009527180c252919ca922 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 20 May 2024 19:35:28 +0300 Subject: [PATCH 091/137] llama : remove Persimmon (#7408) * llama : remove Persimmon * requirements : remove --- README.md | 1 - convert-hf-to-gguf.py | 39 --- convert-persimmon-to-gguf.py | 143 --------- gguf-py/gguf/constants.py | 19 -- llama.cpp | 280 ------------------ requirements.txt | 1 - ...requirements-convert-persimmon-to-gguf.txt | 2 - 7 files changed, 485 deletions(-) delete mode 100755 convert-persimmon-to-gguf.py delete mode 100644 requirements/requirements-convert-persimmon-to-gguf.txt diff --git a/README.md b/README.md index 47d41ebfc..f4088c05e 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,6 @@ Typically finetunes of the base models below are supported as well. - [X] [Aquila 1 & 2](https://huggingface.co/models?search=BAAI/Aquila) - [X] [Starcoder models](https://github.com/ggerganov/llama.cpp/pull/3187) - [X] [Refact](https://huggingface.co/smallcloudai/Refact-1_6B-fim) -- [X] [Persimmon 8B](https://github.com/ggerganov/llama.cpp/pull/3410) - [X] [MPT](https://github.com/ggerganov/llama.cpp/pull/3417) - [X] [Bloom](https://github.com/ggerganov/llama.cpp/pull/3553) - [x] [Yi models](https://huggingface.co/models?search=01-ai/Yi) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index bd303150a..d534b5163 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1148,45 +1148,6 @@ class RefactModel(Model): return tensors -@Model.register("PersimmonForCausalLM") -class PersimmonModel(Model): - model_arch = gguf.MODEL_ARCH.PERSIMMON - - def set_gguf_parameters(self): - block_count = self.hparams.get("num_layers", self.hparams.get("num_hidden_layers")) - head_count = self.hparams["num_attention_heads"] - head_count_kv = head_count - hidden_size = self.hparams["hidden_size"] - - self.gguf_writer.add_name('persimmon-8b-chat') - self.gguf_writer.add_context_length(self.hparams["max_position_embeddings"]) - self.gguf_writer.add_embedding_length(hidden_size) - self.gguf_writer.add_block_count(block_count) - self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"]) - - # NOTE: not sure about this change - why does the model not have a rope dimension count when it is smaller - # than the head size? - # ref: https://github.com/ggerganov/llama.cpp/pull/4889 - # self.gguf_writer.add_rope_dimension_count(hidden_size // head_count) - self.gguf_writer.add_rope_dimension_count(hidden_size // head_count // 2) - - self.gguf_writer.add_head_count(head_count) - self.gguf_writer.add_head_count_kv(head_count_kv) - self.gguf_writer.add_rope_freq_base(self.hparams["rope_theta"]) - self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"]) - - def set_vocab(self): - self._set_vocab_sentencepiece() - # self.gguf_writer.add_bos_token_id(71013) - # self.gguf_writer.add_eos_token_id(71013) - - def extra_f32_tensors(self, name: str, new_name: str, bid: int | None, n_dims: int) -> bool: - del name, new_name, bid, n_dims # unused - - # TODO: FP16 conversion produces garbage outputs. (Q8_0 does not, so..?) - return True - - @Model.register("StableLmForCausalLM", "StableLMEpochForCausalLM", "LlavaStableLMEpochForCausalLM") class StableLMModel(Model): model_arch = gguf.MODEL_ARCH.STABLELM diff --git a/convert-persimmon-to-gguf.py b/convert-persimmon-to-gguf.py deleted file mode 100755 index 07dcade74..000000000 --- a/convert-persimmon-to-gguf.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import annotations - -import logging -import argparse -import os -import sys -from pathlib import Path -from pprint import pprint - -import torch -from sentencepiece import SentencePieceProcessor - -if 'NO_LOCAL_GGUF' not in os.environ: - sys.path.insert(1, str(Path(__file__).parent / 'gguf-py')) -import gguf - -logger = logging.getLogger("persimmon-to-gguf") - - -def _flatten_dict(dct, tensors, prefix=None): - assert isinstance(dct, dict) - for key in dct.keys(): - new_prefix = prefix + '.' + key if prefix is not None else key - if isinstance(dct[key], torch.Tensor): - tensors[new_prefix] = dct[key] - elif isinstance(dct[key], dict): - _flatten_dict(dct[key], tensors, new_prefix) - else: - raise ValueError(type(dct[key])) - return None - - -def _get_sentencepiece_tokenizer_info(dir_model: Path): - tokenizer_path = dir_model / 'adept_vocab.model' - logger.info('getting sentencepiece tokenizer from', tokenizer_path) - tokenizer = SentencePieceProcessor(str(tokenizer_path)) - logger.info('adding tokens') - tokens: list[bytes] = [] - scores: list[float] = [] - toktypes: list[int] = [] - - for i in range(tokenizer.vocab_size()): - text: bytes - score: float - - piece = tokenizer.id_to_piece(i) - text = piece.encode("utf-8") - score = tokenizer.get_score(i) - - toktype = 1 - if tokenizer.is_unknown(i): - toktype = 2 - if tokenizer.is_control(i): - toktype = 3 - if tokenizer.is_unused(i): - toktype = 5 - if tokenizer.is_byte(i): - toktype = 6 - - tokens.append(text) - scores.append(score) - toktypes.append(toktype) - pass - return tokens, scores, toktypes - - -def main(): - parser = argparse.ArgumentParser(description="Convert a Persimmon model from Adept (e.g. Persimmon 8b chat) to a GGML compatible file") - parser.add_argument("--outfile", type=Path, help="path to write to; default: based on input") - parser.add_argument("--ckpt-path", type=Path, help="path to persimmon checkpoint .pt file") - parser.add_argument("--model-dir", type=Path, help="directory containing model e.g. 8b_chat_model_release") - parser.add_argument("--adept-inference-dir", type=str, help="path to adept-inference code directory") - parser.add_argument("--verbose", action="store_true", help="increase output verbosity") - args = parser.parse_args() - logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) - sys.path.append(str(args.adept_inference_dir)) - persimmon_model = torch.load(args.ckpt_path) - hparams = persimmon_model['args'] - pprint(hparams) - tensors: dict[str, torch.Tensor] = {} - _flatten_dict(persimmon_model['model'], tensors, None) - - arch = gguf.MODEL_ARCH.PERSIMMON - gguf_writer = gguf.GGUFWriter(args.outfile, gguf.MODEL_ARCH_NAMES[arch]) - - block_count = hparams.num_layers - head_count = hparams.num_attention_heads - head_count_kv = head_count - ctx_length = hparams.seq_length - hidden_size = hparams.hidden_size - - gguf_writer.add_name('persimmon-8b-chat') - gguf_writer.add_context_length(ctx_length) - gguf_writer.add_embedding_length(hidden_size) - gguf_writer.add_block_count(block_count) - gguf_writer.add_feed_forward_length(hparams.ffn_hidden_size) - # ref: https://github.com/ggerganov/llama.cpp/pull/4889/commits/eea19039fc52ea2dbd1aab45b59ab4e3e29a3443 - gguf_writer.add_rope_dimension_count(hidden_size // head_count // 2) - gguf_writer.add_head_count(head_count) - gguf_writer.add_head_count_kv(head_count_kv) - gguf_writer.add_rope_freq_base(hparams.rotary_emb_base) - gguf_writer.add_layer_norm_eps(hparams.layernorm_epsilon) - - tokens, scores, toktypes = _get_sentencepiece_tokenizer_info(args.model_dir) - gguf_writer.add_tokenizer_model('llama') - gguf_writer.add_tokenizer_pre('default') - gguf_writer.add_token_list(tokens) - gguf_writer.add_token_scores(scores) - gguf_writer.add_token_types(toktypes) - gguf_writer.add_bos_token_id(71013) - gguf_writer.add_eos_token_id(71013) - - tensor_map = gguf.get_tensor_name_map(arch, block_count) - logger.info(tensor_map) - for name in tensors.keys(): - data_torch = tensors[name] - if name.endswith(".self_attention.rotary_emb.inv_freq"): - continue - old_dtype = data_torch.dtype - # TODO: FP16 conversion produces garbage outputs. (Q8_0 does not, so..?) - data = data_torch.to(torch.float32).squeeze().numpy() - new_name = tensor_map.get_name(name, try_suffixes = (".weight", ".bias")) - if new_name is None: - raise ValueError(f"Can not map tensor '{name}'") - - n_dims = len(data.shape) - logger.debug(f"{new_name}, n_dims = {str(n_dims)}, {str(old_dtype)} --> {str(data.dtype)}") - gguf_writer.add_tensor(new_name, data) - logger.info("gguf: write header") - gguf_writer.write_header_to_file() - logger.info("gguf: write metadata") - gguf_writer.write_kv_data_to_file() - logger.info("gguf: write tensors") - gguf_writer.write_tensors_to_file() - - gguf_writer.close() - - logger.info(f"gguf: model successfully exported to '{args.outfile}'") - - -if __name__ == '__main__': - main() diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 978fcada3..692120f4d 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -115,7 +115,6 @@ class MODEL_ARCH(IntEnum): GPTNEOX = auto() MPT = auto() STARCODER = auto() - PERSIMMON = auto() REFACT = auto() BERT = auto() NOMIC_BERT = auto() @@ -193,7 +192,6 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.GPTNEOX: "gptneox", MODEL_ARCH.MPT: "mpt", MODEL_ARCH.STARCODER: "starcoder", - MODEL_ARCH.PERSIMMON: "persimmon", MODEL_ARCH.REFACT: "refact", MODEL_ARCH.BERT: "bert", MODEL_ARCH.NOMIC_BERT: "nomic-bert", @@ -426,20 +424,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], - MODEL_ARCH.PERSIMMON: [ - MODEL_TENSOR.TOKEN_EMBD, - MODEL_TENSOR.OUTPUT, - MODEL_TENSOR.OUTPUT_NORM, - MODEL_TENSOR.ATTN_NORM, - MODEL_TENSOR.ATTN_QKV, - MODEL_TENSOR.ATTN_OUT, - MODEL_TENSOR.FFN_NORM, - MODEL_TENSOR.FFN_DOWN, - MODEL_TENSOR.FFN_UP, - MODEL_TENSOR.ATTN_Q_NORM, - MODEL_TENSOR.ATTN_K_NORM, - MODEL_TENSOR.ATTN_ROT_EMBD, - ], MODEL_ARCH.REFACT: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -756,9 +740,6 @@ MODEL_TENSOR_SKIP: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD, ], - MODEL_ARCH.PERSIMMON: [ - MODEL_TENSOR.ROPE_FREQS, - ], MODEL_ARCH.QWEN: [ MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD, diff --git a/llama.cpp b/llama.cpp index 2025e4558..863961f15 100644 --- a/llama.cpp +++ b/llama.cpp @@ -202,7 +202,6 @@ enum llm_arch { LLM_ARCH_GPTNEOX, LLM_ARCH_MPT, LLM_ARCH_STARCODER, - LLM_ARCH_PERSIMMON, LLM_ARCH_REFACT, LLM_ARCH_BERT, LLM_ARCH_NOMIC_BERT, @@ -239,7 +238,6 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_MPT, "mpt" }, { LLM_ARCH_BAICHUAN, "baichuan" }, { LLM_ARCH_STARCODER, "starcoder" }, - { LLM_ARCH_PERSIMMON, "persimmon" }, { LLM_ARCH_REFACT, "refact" }, { LLM_ARCH_BERT, "bert" }, { LLM_ARCH_NOMIC_BERT, "nomic-bert" }, @@ -595,23 +593,6 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, - { - LLM_ARCH_PERSIMMON, - { - { LLM_TENSOR_TOKEN_EMBD, "token_embd"}, - { LLM_TENSOR_OUTPUT_NORM, "output_norm"}, - { LLM_TENSOR_OUTPUT, "output"}, - { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm"}, - { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv"}, - { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output"}, - { LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm"}, - { LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm"}, - { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm"}, - { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down"}, - { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up"}, - { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd"}, - }, - }, { LLM_ARCH_MPT, { @@ -3967,14 +3948,6 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; - case LLM_ARCH_PERSIMMON: - { - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); - switch (hparams.n_layer) { - case 36: model.type = e_model::MODEL_8B; break; - default: model.type = e_model::MODEL_UNKNOWN; - } - } break; case LLM_ARCH_REFACT: { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -5221,47 +5194,6 @@ static bool llm_load_tensors( layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); } } break; - case LLM_ARCH_PERSIMMON: - { - model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - - { - model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); - } - - for (int i = 0; i < n_layer; ++i) { - ggml_context * ctx_layer = ctx_for_layer(i); - ggml_context * ctx_split = ctx_for_layer_split(i); - - auto & layer = model.layers[i]; - - layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); - layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - - layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); - layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}); - - layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); - - layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); - layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); - - layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); - layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); - - layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); - layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); - - layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {64}); - layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {64}); - - layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {64}); - layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {64}); - } - } break; case LLM_ARCH_BERT: case LLM_ARCH_NOMIC_BERT: { @@ -7923,213 +7855,6 @@ struct llm_build_context { return gf; } - struct ggml_cgraph * build_persimmon() { - struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); - - const int64_t n_embd_head = hparams.n_embd_head_v; - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); - GGML_ASSERT(n_embd_head/2 == hparams.n_rot); - - struct ggml_tensor * cur; - struct ggml_tensor * inpL; - - inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); - - // inp_pos - contains the positions - struct ggml_tensor * inp_pos = build_inp_pos(); - - // KQ_mask (mask for 1 head, it will be broadcasted to all heads) - struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - - for (int il = 0; il < n_layer; ++il) { - struct ggml_tensor * residual = inpL; - - cur = llm_build_norm(ctx0, inpL, hparams, - model.layers[il].attn_norm, - model.layers[il].attn_norm_b, - LLM_NORM, cb, il); - cb(cur, "attn_norm", il); - - // self attention - { - cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur); - cb(cur, "wqkv", il); - - cur = ggml_add(ctx0, cur, model.layers[il].bqkv); - cb(cur, "bqkv", il); - - // split qkv - GGML_ASSERT(n_head_kv == n_head); - - struct ggml_tensor * tmpqkv = ggml_reshape_4d(ctx0, cur, n_embd_head, 3, n_head, n_tokens); - cb(tmpqkv, "tmpqkv", il); - - struct ggml_tensor * tmpqkv_perm = ggml_cont(ctx0, ggml_permute(ctx0, tmpqkv, 0, 3, 1, 2)); - cb(tmpqkv_perm, "tmpqkv", il); - - struct ggml_tensor * tmpq = ggml_view_3d( - ctx0, tmpqkv_perm, n_embd_head, n_head, n_tokens, - ggml_element_size(tmpqkv_perm) * n_embd_head, - ggml_element_size(tmpqkv_perm) * n_embd_head * n_head, - 0 - ); - cb(tmpq, "tmpq", il); - - struct ggml_tensor * tmpk = ggml_view_3d( - ctx0, tmpqkv_perm, n_embd_head, n_head, n_tokens, - ggml_element_size(tmpqkv_perm) * n_embd_head, - ggml_element_size(tmpqkv_perm) * n_embd_head * n_head, - ggml_element_size(tmpqkv_perm) * n_embd_head * n_head * n_tokens - ); - cb(tmpk, "tmpk", il); - - // Q/K Layernorm - tmpq = llm_build_norm(ctx0, tmpq, hparams, - model.layers[il].attn_q_norm, - model.layers[il].attn_q_norm_b, - LLM_NORM, cb, il); - cb(tmpq, "tmpq", il); - - tmpk = llm_build_norm(ctx0, tmpk, hparams, - model.layers[il].attn_k_norm, - model.layers[il].attn_k_norm_b, - LLM_NORM, cb, il); - cb(tmpk, "tmpk", il); - - // RoPE the first n_rot of q/k, pass the other half, and concat. - struct ggml_tensor * qrot = ggml_view_3d( - ctx0, tmpq, n_rot, n_head, n_tokens, - ggml_element_size(tmpq) * n_embd_head, - ggml_element_size(tmpq) * n_embd_head * n_head, - 0 - ); - cb(qrot, "qrot", il); - - struct ggml_tensor * krot = ggml_view_3d( - ctx0, tmpk, n_rot, n_head, n_tokens, - ggml_element_size(tmpk) * n_embd_head, - ggml_element_size(tmpk) * n_embd_head * n_head, - 0 - ); - cb(krot, "krot", il); - - // get the second half of tmpq, e.g tmpq[n_rot:, :, :] - struct ggml_tensor * qpass = ggml_view_3d( - ctx0, tmpq, n_rot, n_head, n_tokens, - ggml_element_size(tmpq) * n_embd_head, - ggml_element_size(tmpq) * n_embd_head * n_head, - ggml_element_size(tmpq) * n_rot - ); - cb(qpass, "qpass", il); - - struct ggml_tensor * kpass = ggml_view_3d( - ctx0, tmpk, n_rot, n_head, n_tokens, - ggml_element_size(tmpk) * n_embd_head, - ggml_element_size(tmpk) * n_embd_head * n_head, - ggml_element_size(tmpk) * n_rot - ); - cb(kpass, "kpass", il); - - struct ggml_tensor * qrotated = ggml_rope_custom( - ctx0, qrot, inp_pos, n_rot, rope_type, 0, n_orig_ctx, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(qrotated, "qrotated", il); - - struct ggml_tensor * krotated = ggml_rope_custom( - ctx0, krot, inp_pos, n_rot, rope_type, 0, n_orig_ctx, - freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow - ); - cb(krotated, "krotated", il); - - // ggml currently only supports concatenation on dim=2 - // so we need to permute qrot, qpass, concat, then permute back. - qrotated = ggml_cont(ctx0, ggml_permute(ctx0, qrotated, 2, 1, 0, 3)); - cb(qrotated, "qrotated", il); - - krotated = ggml_cont(ctx0, ggml_permute(ctx0, krotated, 2, 1, 0, 3)); - cb(krotated, "krotated", il); - - qpass = ggml_cont(ctx0, ggml_permute(ctx0, qpass, 2, 1, 0, 3)); - cb(qpass, "qpass", il); - - kpass = ggml_cont(ctx0, ggml_permute(ctx0, kpass, 2, 1, 0, 3)); - cb(kpass, "kpass", il); - - struct ggml_tensor * Qcur = ggml_concat(ctx0, qrotated, qpass); - cb(Qcur, "Qcur", il); - - struct ggml_tensor * Kcur = ggml_concat(ctx0, krotated, kpass); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Q = ggml_cont(ctx0, ggml_permute(ctx0, Qcur, 2, 1, 0, 3)); - cb(Q, "Q", il); - - Kcur = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 2, 1, 0, 3)); - cb(Kcur, "Kcur", il); - - struct ggml_tensor * Vcur = ggml_view_3d( - ctx0, tmpqkv_perm, n_embd_head, n_head, n_tokens, - ggml_element_size(tmpqkv_perm) * n_embd_head, - ggml_element_size(tmpqkv_perm) * n_embd_head * n_head, - ggml_element_size(tmpqkv_perm) * n_embd_head * n_head * n_tokens * 2 - ); - cb(Vcur, "Vcur", il); - - cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf, - model.layers[il].wo, model.layers[il].bo, - Kcur, Vcur, Q, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); - } - - if (il == n_layer - 1) { - // skip computing output for unused tokens - struct ggml_tensor * inp_out_ids = build_inp_out_ids(); - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - residual = ggml_get_rows(ctx0, residual, inp_out_ids); - } - - struct ggml_tensor * ffn_inp = ggml_add(ctx0, residual, cur); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward network - { - cur = llm_build_norm(ctx0, ffn_inp, hparams, - model.layers[il].ffn_norm, - model.layers[il].ffn_norm_b, - LLM_NORM, cb, il); - cb(cur, "ffn_norm", il); - - cur = llm_build_ffn(ctx0, cur, - model.layers[il].ffn_up, model.layers[il].ffn_up_b, - NULL, NULL, - model.layers[il].ffn_down, model.layers[il].ffn_down_b, - NULL, - LLM_FFN_RELU_SQR, LLM_FFN_SEQ, cb, il); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - cb(cur, "l_out", il); - - inpL = cur; - } - - cur = inpL; - - cur = llm_build_norm(ctx0, cur, hparams, - model.output_norm, - model.output_norm_b, - LLM_NORM, cb, -1); - cb(cur, "result_norm", -1); - - cur = ggml_mul_mat(ctx0, model.output, cur); - cb(cur, "result_output", -1); - - ggml_build_forward_expand(gf, cur); - - return gf; - } - struct ggml_cgraph * build_refact() { struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); @@ -10898,10 +10623,6 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_starcoder(); } break; - case LLM_ARCH_PERSIMMON: - { - result = llm.build_persimmon(); - } break; case LLM_ARCH_REFACT: { result = llm.build_refact(); @@ -15992,7 +15713,6 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_FALCON: case LLM_ARCH_GROK: case LLM_ARCH_DBRX: - case LLM_ARCH_PERSIMMON: case LLM_ARCH_BERT: case LLM_ARCH_NOMIC_BERT: case LLM_ARCH_STABLELM: diff --git a/requirements.txt b/requirements.txt index e7d14e16a..43f82dc2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,3 @@ -r ./requirements/requirements-convert-hf-to-gguf.txt -r ./requirements/requirements-convert-hf-to-gguf-update.txt -r ./requirements/requirements-convert-llama-ggml-to-gguf.txt --r ./requirements/requirements-convert-persimmon-to-gguf.txt diff --git a/requirements/requirements-convert-persimmon-to-gguf.txt b/requirements/requirements-convert-persimmon-to-gguf.txt deleted file mode 100644 index 6ac402610..000000000 --- a/requirements/requirements-convert-persimmon-to-gguf.txt +++ /dev/null @@ -1,2 +0,0 @@ --r ./requirements-convert.txt -torch~=2.1.1 From 917dc8cfa67a72fb7c8bf7392270da3bf4833af4 Mon Sep 17 00:00:00 2001 From: jaime-m-p <167997752+jaime-m-p@users.noreply.github.com> Date: Mon, 20 May 2024 20:15:57 +0200 Subject: [PATCH 092/137] Tokenizer SPM fixes for phi-3 and llama-spm (#7375) * Update brute force test: special tokens * Fix added tokens - Try to read 'added_tokens.json'. - Try to read 'tokenizer_config.json'. - Try to read 'tokenizer.json'. * Fix special tokens rtrim Co-authored-by: Georgi Gerganov * server : fix test regexes --- convert-hf-to-gguf.py | 32 +++++++++++++++++ examples/server/tests/features/server.feature | 10 +++--- .../server/tests/features/slotsave.feature | 4 +-- llama.cpp | 31 +++++++++++++--- tests/test-tokenizer-random.py | 35 +++++++++++++++++-- 5 files changed, 98 insertions(+), 14 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index d534b5163..8937a4981 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1740,6 +1740,38 @@ class Phi3MiniModel(Model): scores[token_id] = -1000.0 toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED + tokenizer_config_file = self.dir_model / 'tokenizer_config.json' + if tokenizer_config_file.is_file(): + with open(tokenizer_config_file, "r", encoding="utf-8") as f: + tokenizer_config_json = json.load(f) + added_tokens_decoder = tokenizer_config_json.get("added_tokens_decoder", {}) + for token_id, foken_data in added_tokens_decoder.items(): + token_id = int(token_id) + token = foken_data["content"].encode("utf-8") + if toktypes[token_id] != SentencePieceTokenTypes.UNKNOWN: + assert(tokens[token_id] == token) + tokens[token_id] = token + scores[token_id] = -1000.0 + toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED + if foken_data.get("special"): + toktypes[token_id] = SentencePieceTokenTypes.CONTROL + + tokenizer_file = self.dir_model / 'tokenizer.json' + if tokenizer_file.is_file(): + with open(tokenizer_file, "r", encoding="utf-8") as f: + tokenizer_json = json.load(f) + added_tokens = tokenizer_json.get("added_tokens", []) + for foken_data in added_tokens: + token_id = int(foken_data["id"]) + token = foken_data["content"].encode("utf-8") + if toktypes[token_id] != SentencePieceTokenTypes.UNKNOWN: + assert(tokens[token_id] == token) + tokens[token_id] = token + scores[token_id] = -1000.0 + toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED + if foken_data.get("special"): + toktypes[token_id] = SentencePieceTokenTypes.CONTROL + self.gguf_writer.add_tokenizer_model("llama") self.gguf_writer.add_tokenizer_pre("default") self.gguf_writer.add_token_list(tokens) diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index d21c09135..048cfad06 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -37,8 +37,8 @@ Feature: llama.cpp server Examples: Prompts | prompt | n_predict | re_content | n_prompt | n_predicted | truncated | - | I believe the meaning of life is | 8 | (read\|going)+ | 18 | 8 | not | - | Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids\|Anna\|forest)+ | 46 | 64 | not | + | I believe the meaning of life is | 8 | (read\|going\|pretty)+ | 18 | 8 | not | + | Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids\|Anna\|forest)+ | 45 | 64 | not | Scenario: Completion prompt truncated Given a prompt: @@ -67,8 +67,8 @@ Feature: llama.cpp server Examples: Prompts | model | system_prompt | user_prompt | max_tokens | re_content | n_prompt | n_predicted | enable_streaming | truncated | - | llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 77 | 8 | disabled | not | - | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird\|Annabyear)+ | -1 | 64 | enabled | | + | llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 76 | 8 | disabled | not | + | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird\|fireplace)+ | -1 | 64 | enabled | | Scenario Outline: OAI Compatibility w/ response format @@ -84,7 +84,7 @@ Feature: llama.cpp server | response_format | n_predicted | re_content | | {"type": "json_object", "schema": {"const": "42"}} | 5 | "42" | | {"type": "json_object", "schema": {"items": [{"type": "integer"}]}} | 10 | \[ -300 \] | - | {"type": "json_object"} | 10 | \{ " Jacky. | + | {"type": "json_object"} | 10 | \{ " Saragine. | Scenario: Tokenize / Detokenize diff --git a/examples/server/tests/features/slotsave.feature b/examples/server/tests/features/slotsave.feature index 1c281c074..ba4ecb6f5 100644 --- a/examples/server/tests/features/slotsave.feature +++ b/examples/server/tests/features/slotsave.feature @@ -26,7 +26,7 @@ Feature: llama.cpp server slot management # Since we have cache, this should only process the last tokens Given a user prompt "What is the capital of Germany?" And a completion request with no api error - Then 24 tokens are predicted matching (Thank|special) + Then 24 tokens are predicted matching (Thank|special|Lily) And 7 prompt tokens are processed # Loading the original cache into slot 0, # we should only be processing 1 prompt token and get the same output @@ -41,7 +41,7 @@ Feature: llama.cpp server slot management Given a user prompt "What is the capital of Germany?" And using slot id 1 And a completion request with no api error - Then 24 tokens are predicted matching (Thank|special) + Then 24 tokens are predicted matching (Thank|special|Lily) And 1 prompt tokens are processed Scenario: Erase Slot diff --git a/llama.cpp b/llama.cpp index 863961f15..e2ebe1752 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4553,7 +4553,8 @@ static void llm_load_vocab( (t.first == "<|eot_id|>" || t.first == "<|im_end|>" || t.first == "<|end|>" || - t.first == "" + t.first == "" || + t.first == "<|endoftext|>" ) ) { vocab.special_eot_id = t.second; @@ -12502,6 +12503,10 @@ static std::vector llama_tokenize_internal(const llama_vocab & output.push_back(vocab.special_bos_id); } + static const bool rtrim = true; //TODO: as param + bool is_prev_special = false; + bool special_token_rtrim = false; + for (const auto & fragment : fragment_buffer) { if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { // without adding this leading whitespace, we do not get the same results as the original tokenizer @@ -12511,9 +12516,21 @@ static std::vector llama_tokenize_internal(const llama_vocab & // and passing 'add space prefix' as bool argument // auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); - if (&fragment == &fragment_buffer.front()) { - if (vocab.add_space_prefix) { - raw_text = " " + raw_text; // prefix with space if the first token is not special + + if (special_token_rtrim) { + size_t num_whitespaces = 0; + while (isspace(raw_text[num_whitespaces])) { + num_whitespaces++; + } + if (num_whitespaces == raw_text.size()) { + continue; // skip if all whitespaces + } + raw_text = raw_text.substr(num_whitespaces); + } + + if (vocab.add_space_prefix) { + if (!output.size() || is_prev_special) { // prefix with space if first token + raw_text = " " + raw_text; } } @@ -12525,6 +12542,12 @@ static std::vector llama_tokenize_internal(const llama_vocab & tokenizer.tokenize(raw_text, output); } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) output.push_back(fragment.token); + is_prev_special = true; + // phi-3 special tokens without rtrim, works fine for llama-spm too + special_token_rtrim = rtrim + && fragment.token != vocab.special_bos_id + && fragment.token != vocab.special_unk_id + && fragment.token != vocab.special_eos_id; } } diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py index d5a6f185f..1166ac1e4 100644 --- a/tests/test-tokenizer-random.py +++ b/tests/test-tokenizer-random.py @@ -153,11 +153,23 @@ def generator_custom_text_edge_cases() -> Iterator[str]: 'Ⅵ-a', # unicode_ranges_digit, {0x00002150, 0x0000218F} // Number Forms '\uFEFF//', # unicode_ranges_control, 0xFEFF (BOM) 'Cửa Việt', # llama-3, ignore_merges = true - 'a', # TODO: Phi-3 fail + 'a', # Phi-3 fail + '<|endoftext|>' # Phi-3 fail 'a\na', # TODO: Bert fail ] +def generator_random_special_tokens(special_tokens:list[str], iterations=100) -> Iterator[str]: + special_tokens = set(special_tokens) + special_tokens.update([" ", "\n", "\t", "-", "!", "one", "1", "", ""]) + special_tokens = list(sorted(special_tokens)) + rand = random.Random() + for m in range(iterations): + rand.seed(m) + words = rand.choices(special_tokens, k=500) + yield "".join(words) + + def generator_vocab_words(vocab: list[str]) -> Iterator[str]: """Brute force check all vocab words""" yield from vocab @@ -289,14 +301,31 @@ def main(argv: list[str] = None): vocab = list(sorted(tokenizer.batch_decode(list(tokenizer.get_vocab().values()), skip_special_tokens=True))) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_custom_text()) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_custom_text_edge_cases()) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_special_tokens(tokenizer.all_special_tokens, 10_000)) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_vocab_words(vocab)) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_chars(10_000)) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_vocab_chars(vocab, 10_000)) - test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_vocab_words(vocab, 10_000)) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_vocab_words(vocab, 5_000)) # test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_bytes(10_000)) # FAIL model.free() if __name__ == "__main__": - main() + # main() + + path_tokenizers = "./models/tokenizers/" + path_vocab_format = "./models/ggml-vocab-%s.gguf" + + # import os + # tokenizers = os.listdir(path_tokenizers) + tokenizers = [ + "llama-spm", # SPM + "phi-3", # SPM + ] + + for tokenizer in tokenizers: + print("\n" + "=" * 50 + "\n" + tokenizer + "\n") # noqa + vocab_file = path_vocab_format % tokenizer + dir_tokenizer = path_tokenizers + "/" + tokenizer + main([vocab_file, dir_tokenizer, "--verbose"]) From d7e852c1bc8e85bf62a6f1aede08cd2de723404a Mon Sep 17 00:00:00 2001 From: jaime-m-p <167997752+jaime-m-p@users.noreply.github.com> Date: Tue, 21 May 2024 14:39:48 +0200 Subject: [PATCH 093/137] Tokenizer SPM fixes for phi-3 and llama-spm (bugfix) (#7425) * Update brute force test: add_special * Update brute force test: default values for add_bos_token and add_eos_token * Enable rtrim when pre-inserting BOS Co-authored-by: Georgi Gerganov * Revert "server : fix test regexes" --- convert-hf-to-gguf.py | 4 ++-- examples/server/tests/features/server.feature | 10 ++++----- .../server/tests/features/slotsave.feature | 4 ++-- llama.cpp | 11 +++++----- tests/test-tokenizer-random.py | 22 +++++++++++-------- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 8937a4981..1acf45bf2 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -1749,7 +1749,7 @@ class Phi3MiniModel(Model): token_id = int(token_id) token = foken_data["content"].encode("utf-8") if toktypes[token_id] != SentencePieceTokenTypes.UNKNOWN: - assert(tokens[token_id] == token) + assert tokens[token_id] == token tokens[token_id] = token scores[token_id] = -1000.0 toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED @@ -1765,7 +1765,7 @@ class Phi3MiniModel(Model): token_id = int(foken_data["id"]) token = foken_data["content"].encode("utf-8") if toktypes[token_id] != SentencePieceTokenTypes.UNKNOWN: - assert(tokens[token_id] == token) + assert tokens[token_id] == token tokens[token_id] = token scores[token_id] = -1000.0 toktypes[token_id] = SentencePieceTokenTypes.USER_DEFINED diff --git a/examples/server/tests/features/server.feature b/examples/server/tests/features/server.feature index 048cfad06..d21c09135 100644 --- a/examples/server/tests/features/server.feature +++ b/examples/server/tests/features/server.feature @@ -37,8 +37,8 @@ Feature: llama.cpp server Examples: Prompts | prompt | n_predict | re_content | n_prompt | n_predicted | truncated | - | I believe the meaning of life is | 8 | (read\|going\|pretty)+ | 18 | 8 | not | - | Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids\|Anna\|forest)+ | 45 | 64 | not | + | I believe the meaning of life is | 8 | (read\|going)+ | 18 | 8 | not | + | Write a joke about AI from a very long prompt which will not be truncated | 256 | (princesses\|everyone\|kids\|Anna\|forest)+ | 46 | 64 | not | Scenario: Completion prompt truncated Given a prompt: @@ -67,8 +67,8 @@ Feature: llama.cpp server Examples: Prompts | model | system_prompt | user_prompt | max_tokens | re_content | n_prompt | n_predicted | enable_streaming | truncated | - | llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 76 | 8 | disabled | not | - | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird\|fireplace)+ | -1 | 64 | enabled | | + | llama-2 | Book | What is the best book | 8 | (Here\|what)+ | 77 | 8 | disabled | not | + | codellama70b | You are a coding assistant. | Write the fibonacci function in c++. | 128 | (thanks\|happy\|bird\|Annabyear)+ | -1 | 64 | enabled | | Scenario Outline: OAI Compatibility w/ response format @@ -84,7 +84,7 @@ Feature: llama.cpp server | response_format | n_predicted | re_content | | {"type": "json_object", "schema": {"const": "42"}} | 5 | "42" | | {"type": "json_object", "schema": {"items": [{"type": "integer"}]}} | 10 | \[ -300 \] | - | {"type": "json_object"} | 10 | \{ " Saragine. | + | {"type": "json_object"} | 10 | \{ " Jacky. | Scenario: Tokenize / Detokenize diff --git a/examples/server/tests/features/slotsave.feature b/examples/server/tests/features/slotsave.feature index ba4ecb6f5..1c281c074 100644 --- a/examples/server/tests/features/slotsave.feature +++ b/examples/server/tests/features/slotsave.feature @@ -26,7 +26,7 @@ Feature: llama.cpp server slot management # Since we have cache, this should only process the last tokens Given a user prompt "What is the capital of Germany?" And a completion request with no api error - Then 24 tokens are predicted matching (Thank|special|Lily) + Then 24 tokens are predicted matching (Thank|special) And 7 prompt tokens are processed # Loading the original cache into slot 0, # we should only be processing 1 prompt token and get the same output @@ -41,7 +41,7 @@ Feature: llama.cpp server slot management Given a user prompt "What is the capital of Germany?" And using slot id 1 And a completion request with no api error - Then 24 tokens are predicted matching (Thank|special|Lily) + Then 24 tokens are predicted matching (Thank|special) And 1 prompt tokens are processed Scenario: Erase Slot diff --git a/llama.cpp b/llama.cpp index e2ebe1752..d26fe559a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -12498,15 +12498,16 @@ static std::vector llama_tokenize_internal(const llama_vocab & // tokenizer.encode('', add_special_tokens=True) returns [1] // tokenizer.encode('', add_special_tokens=False) returns [] - if (add_special && vocab.special_add_bos != 0) { - GGML_ASSERT(vocab.special_bos_id != -1); - output.push_back(vocab.special_bos_id); - } - static const bool rtrim = true; //TODO: as param bool is_prev_special = false; bool special_token_rtrim = false; + if (add_special && vocab.special_add_bos != 0) { + GGML_ASSERT(vocab.special_bos_id != -1); + output.push_back(vocab.special_bos_id); + is_prev_special = true; + } + for (const auto & fragment : fragment_buffer) { if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { // without adding this leading whitespace, we do not get the same results as the original tokenizer diff --git a/tests/test-tokenizer-random.py b/tests/test-tokenizer-random.py index 1166ac1e4..7e1b656e5 100644 --- a/tests/test-tokenizer-random.py +++ b/tests/test-tokenizer-random.py @@ -154,19 +154,22 @@ def generator_custom_text_edge_cases() -> Iterator[str]: '\uFEFF//', # unicode_ranges_control, 0xFEFF (BOM) 'Cửa Việt', # llama-3, ignore_merges = true 'a', # Phi-3 fail - '<|endoftext|>' # Phi-3 fail + '<|endoftext|>', # Phi-3 fail 'a\na', # TODO: Bert fail ] -def generator_random_special_tokens(special_tokens:list[str], iterations=100) -> Iterator[str]: - special_tokens = set(special_tokens) +def generator_random_special_tokens(tokenizer, iterations=100) -> Iterator[str]: + special_tokens = set(tokenizer.all_special_tokens) special_tokens.update([" ", "\n", "\t", "-", "!", "one", "1", "", ""]) special_tokens = list(sorted(special_tokens)) rand = random.Random() for m in range(iterations): rand.seed(m) words = rand.choices(special_tokens, k=500) + if tokenizer.add_bos_token: # skip spam warning of double BOS + while words and words[0] == tokenizer.bos_token: + words.pop(0) yield "".join(words) @@ -290,18 +293,19 @@ def main(argv: list[str] = None): model = LibLlamaModel(LibLlama(), args.vocab_file, mparams=dict(vocab_only=True), cparams=dict(n_ctx=4096)) tokenizer = AutoTokenizer.from_pretrained(args.dir_tokenizer) - def func_tokenize2(text: str): - return tokenizer.encode(text, add_special_tokens=False) - - parse_special = all(len(func_tokenize2(t)) == 1 for t in tokenizer.all_special_tokens) + tokenizer.add_bos_token = getattr(tokenizer, "add_bos_token", True) + tokenizer.add_eos_token = getattr(tokenizer, "add_eos_token", False) def func_tokenize1(text: str): - return model.tokenize(text, add_special=False, parse_special=parse_special) + return model.tokenize(text, add_special=True, parse_special=True) + + def func_tokenize2(text: str): + return tokenizer.encode(text, add_special_tokens=True) vocab = list(sorted(tokenizer.batch_decode(list(tokenizer.get_vocab().values()), skip_special_tokens=True))) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_custom_text()) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_custom_text_edge_cases()) - test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_special_tokens(tokenizer.all_special_tokens, 10_000)) + test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_special_tokens(tokenizer, 10_000)) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_vocab_words(vocab)) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_chars(10_000)) test_compare_tokenizer(func_tokenize1, func_tokenize2, generator_random_vocab_chars(vocab, 10_000)) From d8ee90222791afff2ab666ded4cb6195fd94cced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Tue, 21 May 2024 16:02:12 +0200 Subject: [PATCH 094/137] CUDA: deduplicate mmq code (#7397) --- ggml-cuda/mmq.cu | 1237 ++++++++++------------------------------------ 1 file changed, 271 insertions(+), 966 deletions(-) diff --git a/ggml-cuda/mmq.cu b/ggml-cuda/mmq.cu index 7948f1b12..5b540d375 100644 --- a/ggml-cuda/mmq.cu +++ b/ggml-cuda/mmq.cu @@ -9,6 +9,135 @@ typedef float (*vec_dot_q_mul_mat_cuda_t)( const int * __restrict__ x_ql, const half2 * __restrict__ x_dm, const int * __restrict__ x_qh, const int * __restrict__ x_sc, const int * __restrict__ y_qs, const half2 * __restrict__ y_ms, const int & i, const int & j, const int & k); typedef void (*dot_kernel_k_t)(const void * __restrict__ vx, const int ib, const int iqs, const float * __restrict__ y, float & v); +typedef void (mul_mat_q_t)( + const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, + const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst); + +struct mmq_arch_config_t { + int x; + int y; + int nwarps; +}; + +struct mmq_config_t { + mmq_arch_config_t rdna2; + mmq_arch_config_t rdna1; + mmq_arch_config_t ampere; + mmq_arch_config_t pascal; +}; + +constexpr mmq_config_t MMQ_CONFIG_Q4_0 = { +// x y nwarps + { 64, 128, 8}, + { 64, 64, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + { 64, 128, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q4_1 = { +// x y nwarps + { 64, 128, 8}, + { 64, 64, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + { 64, 128, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q5_0 = { +// x y nwarps + { 64, 128, 8}, + { 64, 64, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + {128, 64, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q5_1 = { +// x y nwarps + { 64, 128, 8}, + { 64, 64, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + {128, 64, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q8_0 = { +// x y nwarps + { 64, 128, 8}, + { 64, 64, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + {128, 64, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q2_K = { +// x y nwarps + { 64, 128, 8}, + {128, 32, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + { 64, 128, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q3_K = { +// x y nwarps + {128, 64, 8}, + { 32, 128, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + {128, 128, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q4_K = { +// x y nwarps + { 64, 128, 8}, + { 32, 64, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + { 64, 128, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q5_K = { +// x y nwarps + { 64, 128, 8}, + { 32, 64, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + { 64, 128, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; +constexpr mmq_config_t MMQ_CONFIG_Q6_K = { +// x y nwarps + { 64, 128, 8}, + { 32, 64, 8}, +#ifdef CUDA_USE_TENSOR_CORES + { 4, 32, 4}, +#else + { 64, 64, 4}, +#endif // CUDA_USE_TENSOR_CORES + { 64, 64, 8}, +}; + +// ------------------------------------------------------------ template static __device__ __forceinline__ void allocate_tiles_q4_0(int ** x_ql, half2 ** x_dm, int ** x_qh, int ** x_sc) { GGML_UNUSED(x_qh); @@ -943,25 +1072,6 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_mul_mat( return vec_dot_q6_K_q8_1_impl_mmq(&x_ql[index_x], &y_qs[index_y], sc, x_dmf[i * (WARP_SIZE/QI6_K) + i/QI6_K], &y_df[index_y/QI8_1]); } -#define MMQ_X_Q4_0_RDNA2 64 -#define MMQ_Y_Q4_0_RDNA2 128 -#define NWARPS_Q4_0_RDNA2 8 -#define MMQ_X_Q4_0_RDNA1 64 -#define MMQ_Y_Q4_0_RDNA1 64 -#define NWARPS_Q4_0_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q4_0_AMPERE 4 -#define MMQ_Y_Q4_0_AMPERE 32 -#define NWARPS_Q4_0_AMPERE 4 -#else -#define MMQ_X_Q4_0_AMPERE 64 -#define MMQ_Y_Q4_0_AMPERE 128 -#define NWARPS_Q4_0_AMPERE 4 -#endif -#define MMQ_X_Q4_0_PASCAL 64 -#define MMQ_Y_Q4_0_PASCAL 64 -#define NWARPS_Q4_0_PASCAL 8 - template static __device__ __forceinline__ void mul_mat_q( @@ -1072,1107 +1182,265 @@ static __device__ __forceinline__ void mul_mat_q( } } +static constexpr __device__ mmq_arch_config_t get_arch_config_device(mmq_config_t mmq_config) { + +#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) + +#if defined(RDNA3) || defined(RDNA2) + return mmq_config.rdna2; +#else + return mmq_config.rdna1; +#endif // defined(RDNA3) || defined(RDNA2) + +#else + +#if __CUDA_ARCH__ >= CC_VOLTA + return mmq_config.ampere; +#else + return mmq_config.pascal; +#endif // __CUDA_ARCH__ >= CC_VOLTA + +#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) +} + template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q4_0_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_0.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) mul_mat_q4_0( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q4_0_RDNA2; - const int mmq_y = MMQ_Y_Q4_0_RDNA2; - const int nwarps = NWARPS_Q4_0_RDNA2; -#else - const int mmq_x = MMQ_X_Q4_0_RDNA1; - const int mmq_y = MMQ_Y_Q4_0_RDNA1; - const int nwarps = NWARPS_Q4_0_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q4_0); - mul_mat_q, - load_tiles_q4_0, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q4_0_AMPERE; - const int mmq_y = MMQ_Y_Q4_0_AMPERE; - const int nwarps = NWARPS_Q4_0_AMPERE; - - mul_mat_q, - load_tiles_q4_0, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q4_0_PASCAL; - const int mmq_y = MMQ_Y_Q4_0_PASCAL; - const int nwarps = NWARPS_Q4_0_PASCAL; - - mul_mat_q, - load_tiles_q4_0, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat> + mul_mat_q, + load_tiles_q4_0, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q4_0_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q4_1_RDNA2 64 -#define MMQ_Y_Q4_1_RDNA2 128 -#define NWARPS_Q4_1_RDNA2 8 -#define MMQ_X_Q4_1_RDNA1 64 -#define MMQ_Y_Q4_1_RDNA1 64 -#define NWARPS_Q4_1_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q4_1_AMPERE 4 -#define MMQ_Y_Q4_1_AMPERE 32 -#define NWARPS_Q4_1_AMPERE 4 -#else -#define MMQ_X_Q4_1_AMPERE 64 -#define MMQ_Y_Q4_1_AMPERE 128 -#define NWARPS_Q4_1_AMPERE 4 -#endif -#define MMQ_X_Q4_1_PASCAL 64 -#define MMQ_Y_Q4_1_PASCAL 64 -#define NWARPS_Q4_1_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q4_1_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_1.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #elif __CUDA_ARCH__ < CC_VOLTA - __launch_bounds__(WARP_SIZE*NWARPS_Q4_1_PASCAL, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_1.pascal.nwarps, 2) #endif // __CUDA_ARCH__ < CC_VOLTA mul_mat_q4_1( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q4_1_RDNA2; - const int mmq_y = MMQ_Y_Q4_1_RDNA2; - const int nwarps = NWARPS_Q4_1_RDNA2; -#else - const int mmq_x = MMQ_X_Q4_1_RDNA1; - const int mmq_y = MMQ_Y_Q4_1_RDNA1; - const int nwarps = NWARPS_Q4_1_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q4_1); - mul_mat_q, - load_tiles_q4_1, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q4_1_AMPERE; - const int mmq_y = MMQ_Y_Q4_1_AMPERE; - const int nwarps = NWARPS_Q4_1_AMPERE; - - mul_mat_q, - load_tiles_q4_1, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q4_1_PASCAL; - const int mmq_y = MMQ_Y_Q4_1_PASCAL; - const int nwarps = NWARPS_Q4_1_PASCAL; - - mul_mat_q, - load_tiles_q4_1, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat> + mul_mat_q, + load_tiles_q4_1, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q4_1_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q5_0_RDNA2 64 -#define MMQ_Y_Q5_0_RDNA2 128 -#define NWARPS_Q5_0_RDNA2 8 -#define MMQ_X_Q5_0_RDNA1 64 -#define MMQ_Y_Q5_0_RDNA1 64 -#define NWARPS_Q5_0_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q5_0_AMPERE 4 -#define MMQ_Y_Q5_0_AMPERE 32 -#define NWARPS_Q5_0_AMPERE 4 -#else -#define MMQ_X_Q5_0_AMPERE 128 -#define MMQ_Y_Q5_0_AMPERE 64 -#define NWARPS_Q5_0_AMPERE 4 -#endif -#define MMQ_X_Q5_0_PASCAL 64 -#define MMQ_Y_Q5_0_PASCAL 64 -#define NWARPS_Q5_0_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q5_0_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q5_0.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) mul_mat_q5_0( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q5_0_RDNA2; - const int mmq_y = MMQ_Y_Q5_0_RDNA2; - const int nwarps = NWARPS_Q5_0_RDNA2; -#else - const int mmq_x = MMQ_X_Q5_0_RDNA1; - const int mmq_y = MMQ_Y_Q5_0_RDNA1; - const int nwarps = NWARPS_Q5_0_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q5_0); - mul_mat_q, - load_tiles_q5_0, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q5_0_AMPERE; - const int mmq_y = MMQ_Y_Q5_0_AMPERE; - const int nwarps = NWARPS_Q5_0_AMPERE; - - mul_mat_q, - load_tiles_q5_0, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q5_0_PASCAL; - const int mmq_y = MMQ_Y_Q5_0_PASCAL; - const int nwarps = NWARPS_Q5_0_PASCAL; - - mul_mat_q, - load_tiles_q5_0, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat> + mul_mat_q, + load_tiles_q5_0, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q5_0_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q5_1_RDNA2 64 -#define MMQ_Y_Q5_1_RDNA2 128 -#define NWARPS_Q5_1_RDNA2 8 -#define MMQ_X_Q5_1_RDNA1 64 -#define MMQ_Y_Q5_1_RDNA1 64 -#define NWARPS_Q5_1_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q5_1_AMPERE 4 -#define MMQ_Y_Q5_1_AMPERE 32 -#define NWARPS_Q5_1_AMPERE 4 -#else -#define MMQ_X_Q5_1_AMPERE 128 -#define MMQ_Y_Q5_1_AMPERE 64 -#define NWARPS_Q5_1_AMPERE 4 -#endif -#define MMQ_X_Q5_1_PASCAL 64 -#define MMQ_Y_Q5_1_PASCAL 64 -#define NWARPS_Q5_1_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q5_1_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q5_1.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) mul_mat_q5_1( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q5_1_RDNA2; - const int mmq_y = MMQ_Y_Q5_1_RDNA2; - const int nwarps = NWARPS_Q5_1_RDNA2; -#else - const int mmq_x = MMQ_X_Q5_1_RDNA1; - const int mmq_y = MMQ_Y_Q5_1_RDNA1; - const int nwarps = NWARPS_Q5_1_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q5_1); - mul_mat_q, - load_tiles_q5_1, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q5_1_AMPERE; - const int mmq_y = MMQ_Y_Q5_1_AMPERE; - const int nwarps = NWARPS_Q5_1_AMPERE; - - mul_mat_q, - load_tiles_q5_1, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q5_1_PASCAL; - const int mmq_y = MMQ_Y_Q5_1_PASCAL; - const int nwarps = NWARPS_Q5_1_PASCAL; - - mul_mat_q, - load_tiles_q5_1, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat> + mul_mat_q, + load_tiles_q5_1, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q5_1_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q8_0_RDNA2 64 -#define MMQ_Y_Q8_0_RDNA2 128 -#define NWARPS_Q8_0_RDNA2 8 -#define MMQ_X_Q8_0_RDNA1 64 -#define MMQ_Y_Q8_0_RDNA1 64 -#define NWARPS_Q8_0_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q8_0_AMPERE 4 -#define MMQ_Y_Q8_0_AMPERE 32 -#define NWARPS_Q8_0_AMPERE 4 -#else -#define MMQ_X_Q8_0_AMPERE 128 -#define MMQ_Y_Q8_0_AMPERE 64 -#define NWARPS_Q8_0_AMPERE 4 -#endif -#define MMQ_X_Q8_0_PASCAL 64 -#define MMQ_Y_Q8_0_PASCAL 64 -#define NWARPS_Q8_0_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q8_0_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q8_0.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) mul_mat_q8_0( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q8_0_RDNA2; - const int mmq_y = MMQ_Y_Q8_0_RDNA2; - const int nwarps = NWARPS_Q8_0_RDNA2; -#else - const int mmq_x = MMQ_X_Q8_0_RDNA1; - const int mmq_y = MMQ_Y_Q8_0_RDNA1; - const int nwarps = NWARPS_Q8_0_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q8_0); - mul_mat_q, - load_tiles_q8_0, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q8_0_AMPERE; - const int mmq_y = MMQ_Y_Q8_0_AMPERE; - const int nwarps = NWARPS_Q8_0_AMPERE; - - mul_mat_q, - load_tiles_q8_0, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q8_0_PASCAL; - const int mmq_y = MMQ_Y_Q8_0_PASCAL; - const int nwarps = NWARPS_Q8_0_PASCAL; - - mul_mat_q, - load_tiles_q8_0, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat> + mul_mat_q, + load_tiles_q8_0, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q8_0_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q2_K_RDNA2 64 -#define MMQ_Y_Q2_K_RDNA2 128 -#define NWARPS_Q2_K_RDNA2 8 -#define MMQ_X_Q2_K_RDNA1 128 -#define MMQ_Y_Q2_K_RDNA1 32 -#define NWARPS_Q2_K_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q2_K_AMPERE 4 -#define MMQ_Y_Q2_K_AMPERE 32 -#define NWARPS_Q2_K_AMPERE 4 -#else -#define MMQ_X_Q2_K_AMPERE 64 -#define MMQ_Y_Q2_K_AMPERE 128 -#define NWARPS_Q2_K_AMPERE 4 -#endif -#define MMQ_X_Q2_K_PASCAL 64 -#define MMQ_Y_Q2_K_PASCAL 64 -#define NWARPS_Q2_K_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q2_K_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q2_K.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) mul_mat_q2_K( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q2_K_RDNA2; - const int mmq_y = MMQ_Y_Q2_K_RDNA2; - const int nwarps = NWARPS_Q2_K_RDNA2; -#else - const int mmq_x = MMQ_X_Q2_K_RDNA1; - const int mmq_y = MMQ_Y_Q2_K_RDNA1; - const int nwarps = NWARPS_Q2_K_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q2_K); - mul_mat_q, - load_tiles_q2_K, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q2_K_AMPERE; - const int mmq_y = MMQ_Y_Q2_K_AMPERE; - const int nwarps = NWARPS_Q2_K_AMPERE; - - mul_mat_q, - load_tiles_q2_K, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q2_K_PASCAL; - const int mmq_y = MMQ_Y_Q2_K_PASCAL; - const int nwarps = NWARPS_Q2_K_PASCAL; - - mul_mat_q, - load_tiles_q2_K, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat> + mul_mat_q, + load_tiles_q2_K, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q2_K_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q3_K_RDNA2 128 -#define MMQ_Y_Q3_K_RDNA2 64 -#define NWARPS_Q3_K_RDNA2 8 -#define MMQ_X_Q3_K_RDNA1 32 -#define MMQ_Y_Q3_K_RDNA1 128 -#define NWARPS_Q3_K_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q3_K_AMPERE 4 -#define MMQ_Y_Q3_K_AMPERE 32 -#define NWARPS_Q3_K_AMPERE 4 -#else -#define MMQ_X_Q3_K_AMPERE 128 -#define MMQ_Y_Q3_K_AMPERE 128 -#define NWARPS_Q3_K_AMPERE 4 -#endif -#define MMQ_X_Q3_K_PASCAL 64 -#define MMQ_Y_Q3_K_PASCAL 64 -#define NWARPS_Q3_K_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q3_K_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q3_K.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #elif __CUDA_ARCH__ < CC_VOLTA - __launch_bounds__(WARP_SIZE*NWARPS_Q3_K_PASCAL, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q3_K.pascal.nwarps, 2) #endif // __CUDA_ARCH__ < CC_VOLTA mul_mat_q3_K( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q3_K_RDNA2; - const int mmq_y = MMQ_Y_Q3_K_RDNA2; - const int nwarps = NWARPS_Q3_K_RDNA2; -#else - const int mmq_x = MMQ_X_Q3_K_RDNA1; - const int mmq_y = MMQ_Y_Q3_K_RDNA1; - const int nwarps = NWARPS_Q3_K_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q3_K); - mul_mat_q, - load_tiles_q3_K, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q3_K_AMPERE; - const int mmq_y = MMQ_Y_Q3_K_AMPERE; - const int nwarps = NWARPS_Q3_K_AMPERE; - - mul_mat_q, - load_tiles_q3_K, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q3_K_PASCAL; - const int mmq_y = MMQ_Y_Q3_K_PASCAL; - const int nwarps = NWARPS_Q3_K_PASCAL; - - mul_mat_q, - load_tiles_q3_K, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat> + mul_mat_q, + load_tiles_q3_K, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q3_K_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q4_K_RDNA2 64 -#define MMQ_Y_Q4_K_RDNA2 128 -#define NWARPS_Q4_K_RDNA2 8 -#define MMQ_X_Q4_K_RDNA1 32 -#define MMQ_Y_Q4_K_RDNA1 64 -#define NWARPS_Q4_K_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q4_K_AMPERE 4 -#define MMQ_Y_Q4_K_AMPERE 32 -#define NWARPS_Q4_K_AMPERE 4 -#else -#define MMQ_X_Q4_K_AMPERE 64 -#define MMQ_Y_Q4_K_AMPERE 128 -#define NWARPS_Q4_K_AMPERE 4 -#endif -#define MMQ_X_Q4_K_PASCAL 64 -#define MMQ_Y_Q4_K_PASCAL 64 -#define NWARPS_Q4_K_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q4_K_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_K.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #elif __CUDA_ARCH__ < CC_VOLTA - __launch_bounds__(WARP_SIZE*NWARPS_Q4_K_PASCAL, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_K.pascal.nwarps, 2) #endif // __CUDA_ARCH__ < CC_VOLTA mul_mat_q4_K( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q4_K_RDNA2; - const int mmq_y = MMQ_Y_Q4_K_RDNA2; - const int nwarps = NWARPS_Q4_K_RDNA2; -#else - const int mmq_x = MMQ_X_Q4_K_RDNA1; - const int mmq_y = MMQ_Y_Q4_K_RDNA1; - const int nwarps = NWARPS_Q4_K_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q4_K); - mul_mat_q, - load_tiles_q4_K, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q4_K_AMPERE; - const int mmq_y = MMQ_Y_Q4_K_AMPERE; - const int nwarps = NWARPS_Q4_K_AMPERE; - - mul_mat_q, - load_tiles_q4_K, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q4_K_PASCAL; - const int mmq_y = MMQ_Y_Q4_K_PASCAL; - const int nwarps = NWARPS_Q4_K_PASCAL; - - mul_mat_q, - load_tiles_q4_K, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat> + mul_mat_q, + load_tiles_q4_K, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q4_K_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q5_K_RDNA2 64 -#define MMQ_Y_Q5_K_RDNA2 128 -#define NWARPS_Q5_K_RDNA2 8 -#define MMQ_X_Q5_K_RDNA1 32 -#define MMQ_Y_Q5_K_RDNA1 64 -#define NWARPS_Q5_K_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q5_K_AMPERE 4 -#define MMQ_Y_Q5_K_AMPERE 32 -#define NWARPS_Q5_K_AMPERE 4 -#else -#define MMQ_X_Q5_K_AMPERE 64 -#define MMQ_Y_Q5_K_AMPERE 128 -#define NWARPS_Q5_K_AMPERE 4 -#endif -#define MMQ_X_Q5_K_PASCAL 64 -#define MMQ_Y_Q5_K_PASCAL 64 -#define NWARPS_Q5_K_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q5_K_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q5_K.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) mul_mat_q5_K( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q5_K_RDNA2; - const int mmq_y = MMQ_Y_Q5_K_RDNA2; - const int nwarps = NWARPS_Q5_K_RDNA2; -#else - const int mmq_x = MMQ_X_Q5_K_RDNA1; - const int mmq_y = MMQ_Y_Q5_K_RDNA1; - const int nwarps = NWARPS_Q5_K_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q5_K); - mul_mat_q, - load_tiles_q5_K, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q5_K_AMPERE; - const int mmq_y = MMQ_Y_Q5_K_AMPERE; - const int nwarps = NWARPS_Q5_K_AMPERE; - - mul_mat_q, - load_tiles_q5_K, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q5_K_PASCAL; - const int mmq_y = MMQ_Y_Q5_K_PASCAL; - const int nwarps = NWARPS_Q5_K_PASCAL; - - mul_mat_q, - load_tiles_q5_K, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat> + mul_mat_q, + load_tiles_q5_K, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q5_K_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -#define MMQ_X_Q6_K_RDNA2 64 -#define MMQ_Y_Q6_K_RDNA2 128 -#define NWARPS_Q6_K_RDNA2 8 -#define MMQ_X_Q6_K_RDNA1 32 -#define MMQ_Y_Q6_K_RDNA1 64 -#define NWARPS_Q6_K_RDNA1 8 -#if defined(CUDA_USE_TENSOR_CORES) -#define MMQ_X_Q6_K_AMPERE 4 -#define MMQ_Y_Q6_K_AMPERE 32 -#define NWARPS_Q6_K_AMPERE 4 -#else -#define MMQ_X_Q6_K_AMPERE 64 -#define MMQ_Y_Q6_K_AMPERE 64 -#define NWARPS_Q6_K_AMPERE 4 -#endif -#define MMQ_X_Q6_K_PASCAL 64 -#define MMQ_Y_Q6_K_PASCAL 64 -#define NWARPS_Q6_K_PASCAL 8 - template static __global__ void #if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) #if defined(RDNA3) || defined(RDNA2) - __launch_bounds__(WARP_SIZE*NWARPS_Q6_K_RDNA2, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q6_K.rdna2.nwarps, 2) #endif // defined(RDNA3) || defined(RDNA2) #elif __CUDA_ARCH__ < CC_VOLTA - __launch_bounds__(WARP_SIZE*NWARPS_Q6_K_PASCAL, 2) + __launch_bounds__(WARP_SIZE*MMQ_CONFIG_Q4_K.pascal.nwarps, 2) #endif // __CUDA_ARCH__ < CC_VOLTA mul_mat_q6_K( const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols_x, const int nrows_x, const int ncols_y, const int nrows_y, const int nrows_dst) { -#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__) -#if defined(RDNA3) || defined(RDNA2) - const int mmq_x = MMQ_X_Q6_K_RDNA2; - const int mmq_y = MMQ_Y_Q6_K_RDNA2; - const int nwarps = NWARPS_Q6_K_RDNA2; -#else - const int mmq_x = MMQ_X_Q6_K_RDNA1; - const int mmq_y = MMQ_Y_Q6_K_RDNA1; - const int nwarps = NWARPS_Q6_K_RDNA1; -#endif // defined(RDNA3) || defined(RDNA2) +#if __CUDA_ARCH__ >= MIN_CC_DP4A + constexpr mmq_arch_config_t arch_config = get_arch_config_device(MMQ_CONFIG_Q6_K); - mul_mat_q, - load_tiles_q6_K, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= CC_VOLTA - const int mmq_x = MMQ_X_Q6_K_AMPERE; - const int mmq_y = MMQ_Y_Q6_K_AMPERE; - const int nwarps = NWARPS_Q6_K_AMPERE; - - mul_mat_q, - load_tiles_q6_K, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - -#elif __CUDA_ARCH__ >= MIN_CC_DP4A - const int mmq_x = MMQ_X_Q6_K_PASCAL; - const int mmq_y = MMQ_Y_Q6_K_PASCAL; - const int nwarps = NWARPS_Q6_K_PASCAL; - - mul_mat_q, - load_tiles_q6_K, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat> + mul_mat_q, + load_tiles_q6_K, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else GGML_UNUSED(vec_dot_q6_K_q8_1_mul_mat); NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= CC_VOLTA +#endif // __CUDA_ARCH__ >= MIN_CC_DP4A } -static void ggml_mul_mat_q4_0_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q4_0_RDNA2; - mmq_y = MMQ_Y_Q4_0_RDNA2; - nwarps = NWARPS_Q4_0_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q4_0_RDNA1; - mmq_y = MMQ_Y_Q4_0_RDNA1; - nwarps = NWARPS_Q4_0_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q4_0_AMPERE; - mmq_y = MMQ_Y_Q4_0_AMPERE; - nwarps = NWARPS_Q4_0_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q4_0_PASCAL; - mmq_y = MMQ_Y_Q4_0_PASCAL; - nwarps = NWARPS_Q4_0_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q4_0<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q4_0<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} - -static void ggml_mul_mat_q4_1_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q4_1_RDNA2; - mmq_y = MMQ_Y_Q4_1_RDNA2; - nwarps = NWARPS_Q4_1_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q4_1_RDNA1; - mmq_y = MMQ_Y_Q4_1_RDNA1; - nwarps = NWARPS_Q4_1_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q4_1_AMPERE; - mmq_y = MMQ_Y_Q4_1_AMPERE; - nwarps = NWARPS_Q4_1_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q4_1_PASCAL; - mmq_y = MMQ_Y_Q4_1_PASCAL; - nwarps = NWARPS_Q4_1_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q4_1<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q4_1<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} - -static void ggml_mul_mat_q5_0_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q5_0_RDNA2; - mmq_y = MMQ_Y_Q5_0_RDNA2; - nwarps = NWARPS_Q5_0_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q5_0_RDNA1; - mmq_y = MMQ_Y_Q5_0_RDNA1; - nwarps = NWARPS_Q5_0_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q5_0_AMPERE; - mmq_y = MMQ_Y_Q5_0_AMPERE; - nwarps = NWARPS_Q5_0_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q5_0_PASCAL; - mmq_y = MMQ_Y_Q5_0_PASCAL; - nwarps = NWARPS_Q5_0_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q5_0<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q5_0<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} - -static void ggml_mul_mat_q5_1_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q5_1_RDNA2; - mmq_y = MMQ_Y_Q5_1_RDNA2; - nwarps = NWARPS_Q5_1_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q5_1_RDNA1; - mmq_y = MMQ_Y_Q5_1_RDNA1; - nwarps = NWARPS_Q5_1_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q5_1_AMPERE; - mmq_y = MMQ_Y_Q5_1_AMPERE; - nwarps = NWARPS_Q5_1_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q5_1_PASCAL; - mmq_y = MMQ_Y_Q5_1_PASCAL; - nwarps = NWARPS_Q5_1_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q5_1<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q5_1<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} - -static void ggml_mul_mat_q8_0_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q8_0_RDNA2; - mmq_y = MMQ_Y_Q8_0_RDNA2; - nwarps = NWARPS_Q8_0_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q8_0_RDNA1; - mmq_y = MMQ_Y_Q8_0_RDNA1; - nwarps = NWARPS_Q8_0_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q8_0_AMPERE; - mmq_y = MMQ_Y_Q8_0_AMPERE; - nwarps = NWARPS_Q8_0_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q8_0_PASCAL; - mmq_y = MMQ_Y_Q8_0_PASCAL; - nwarps = NWARPS_Q8_0_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q8_0<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q8_0<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} - -static void ggml_mul_mat_q2_K_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q2_K_RDNA2; - mmq_y = MMQ_Y_Q2_K_RDNA2; - nwarps = NWARPS_Q2_K_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q2_K_RDNA1; - mmq_y = MMQ_Y_Q2_K_RDNA1; - nwarps = NWARPS_Q2_K_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q2_K_AMPERE; - mmq_y = MMQ_Y_Q2_K_AMPERE; - nwarps = NWARPS_Q2_K_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q2_K_PASCAL; - mmq_y = MMQ_Y_Q2_K_PASCAL; - nwarps = NWARPS_Q2_K_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q2_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q2_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} - -static void ggml_mul_mat_q3_K_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - -#if QK_K == 256 - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q3_K_RDNA2; - mmq_y = MMQ_Y_Q3_K_RDNA2; - nwarps = NWARPS_Q3_K_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q3_K_RDNA1; - mmq_y = MMQ_Y_Q3_K_RDNA1; - nwarps = NWARPS_Q3_K_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q3_K_AMPERE; - mmq_y = MMQ_Y_Q3_K_AMPERE; - nwarps = NWARPS_Q3_K_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q3_K_PASCAL; - mmq_y = MMQ_Y_Q3_K_PASCAL; - nwarps = NWARPS_Q3_K_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q3_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q3_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -#endif -} - -static void ggml_mul_mat_q4_K_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q4_K_RDNA2; - mmq_y = MMQ_Y_Q4_K_RDNA2; - nwarps = NWARPS_Q4_K_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q4_K_RDNA1; - mmq_y = MMQ_Y_Q4_K_RDNA1; - nwarps = NWARPS_Q4_K_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q4_K_AMPERE; - mmq_y = MMQ_Y_Q4_K_AMPERE; - nwarps = NWARPS_Q4_K_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q4_K_PASCAL; - mmq_y = MMQ_Y_Q4_K_PASCAL; - nwarps = NWARPS_Q4_K_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q4_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q4_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} - -static void ggml_mul_mat_q5_K_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q5_K_RDNA2; - mmq_y = MMQ_Y_Q5_K_RDNA2; - nwarps = NWARPS_Q5_K_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q5_K_RDNA1; - mmq_y = MMQ_Y_Q5_K_RDNA1; - nwarps = NWARPS_Q5_K_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q5_K_AMPERE; - mmq_y = MMQ_Y_Q5_K_AMPERE; - nwarps = NWARPS_Q5_K_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q5_K_PASCAL; - mmq_y = MMQ_Y_Q5_K_PASCAL; - nwarps = NWARPS_Q5_K_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q5_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q5_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} - -static void ggml_mul_mat_q6_K_q8_1_cuda( - const void * vx, const void * vy, float * dst, const int ncols_x, const int nrows_x, - const int ncols_y, const int nrows_y, const int nrows_dst, cudaStream_t stream) { - - int id = ggml_cuda_get_device(); - const int compute_capability = ggml_cuda_info().devices[id].cc; - - int mmq_x, mmq_y, nwarps; - if (compute_capability >= CC_RDNA2) { - mmq_x = MMQ_X_Q6_K_RDNA2; - mmq_y = MMQ_Y_Q6_K_RDNA2; - nwarps = NWARPS_Q6_K_RDNA2; - } else if (compute_capability >= CC_OFFSET_AMD) { - mmq_x = MMQ_X_Q6_K_RDNA1; - mmq_y = MMQ_Y_Q6_K_RDNA1; - nwarps = NWARPS_Q6_K_RDNA1; - } else if (compute_capability >= CC_VOLTA) { - mmq_x = MMQ_X_Q6_K_AMPERE; - mmq_y = MMQ_Y_Q6_K_AMPERE; - nwarps = NWARPS_Q6_K_AMPERE; - } else if (compute_capability >= MIN_CC_DP4A) { - mmq_x = MMQ_X_Q6_K_PASCAL; - mmq_y = MMQ_Y_Q6_K_PASCAL; - nwarps = NWARPS_Q6_K_PASCAL; - } else { - GGML_ASSERT(false); - } - - const int block_num_x = (nrows_x + mmq_y - 1) / mmq_y; - const int block_num_y = (ncols_y + mmq_x - 1) / mmq_x; - const dim3 block_nums(block_num_x, block_num_y, 1); - const dim3 block_dims(WARP_SIZE, nwarps, 1); - - if (nrows_x % mmq_y == 0) { - const bool need_check = false; - mul_mat_q6_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } else { - const bool need_check = true; - mul_mat_q6_K<<>> - (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); - } -} +#define MMQ_SWITCH_CASE(type_suffix) \ + case GGML_TYPE_Q##type_suffix: if (row_diff % arch_config.y == 0) { \ + const bool need_check = false; \ + mul_mat_q##type_suffix<<>> \ + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst); \ + } else { \ + const bool need_check = true; \ + mul_mat_q##type_suffix<<>> \ + (src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst); \ + } break; \ void ggml_cuda_op_mul_mat_q( ggml_backend_cuda_context & ctx, @@ -2190,47 +1458,84 @@ void ggml_cuda_op_mul_mat_q( const int64_t row_diff = row_high - row_low; int id = ggml_cuda_get_device(); + const int compute_capability = ggml_cuda_info().devices[id].cc; // the main device has a larger memory buffer to hold the results from all GPUs // nrows_dst == nrows of the matrix that the kernel writes into const int64_t nrows_dst = id == ctx.device ? ne0 : row_diff; + mmq_config_t mmq_config; + switch (src0->type) { case GGML_TYPE_Q4_0: - ggml_mul_mat_q4_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q4_0; break; case GGML_TYPE_Q4_1: - ggml_mul_mat_q4_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q4_1; break; case GGML_TYPE_Q5_0: - ggml_mul_mat_q5_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q5_0; break; case GGML_TYPE_Q5_1: - ggml_mul_mat_q5_1_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q5_1; break; case GGML_TYPE_Q8_0: - ggml_mul_mat_q8_0_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q8_0; break; case GGML_TYPE_Q2_K: - ggml_mul_mat_q2_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q2_K; break; case GGML_TYPE_Q3_K: - ggml_mul_mat_q3_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q3_K; break; case GGML_TYPE_Q4_K: - ggml_mul_mat_q4_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q4_K; break; case GGML_TYPE_Q5_K: - ggml_mul_mat_q5_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q5_K; break; case GGML_TYPE_Q6_K: - ggml_mul_mat_q6_K_q8_1_cuda(src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, src1_ncols, src1_padded_row_size, nrows_dst, stream); + mmq_config = MMQ_CONFIG_Q6_K; break; default: GGML_ASSERT(false); break; } + mmq_arch_config_t arch_config; + if (compute_capability >= CC_RDNA2) { + arch_config = mmq_config.rdna2; + } else if (compute_capability >= CC_OFFSET_AMD) { + arch_config = mmq_config.rdna1; + } else if (compute_capability >= CC_VOLTA) { + arch_config = mmq_config.ampere; + } else if (compute_capability >= MIN_CC_DP4A) { + arch_config = mmq_config.pascal; + } else { + GGML_ASSERT(false); + } + + const int block_num_x = (row_diff + arch_config.y - 1) / arch_config.y; + const int block_num_y = (src1_ncols + arch_config.x - 1) / arch_config.x; + const dim3 block_nums(block_num_x, block_num_y, 1); + const dim3 block_dims(WARP_SIZE, arch_config.nwarps, 1); + + switch (src0->type) { + MMQ_SWITCH_CASE(4_0) + MMQ_SWITCH_CASE(4_1) + MMQ_SWITCH_CASE(5_0) + MMQ_SWITCH_CASE(5_1) + MMQ_SWITCH_CASE(8_0) + MMQ_SWITCH_CASE(2_K) + MMQ_SWITCH_CASE(3_K) + MMQ_SWITCH_CASE(4_K) + MMQ_SWITCH_CASE(5_K) + MMQ_SWITCH_CASE(6_K) + default: + GGML_ASSERT(false); + break; + } + GGML_UNUSED(src1); GGML_UNUSED(dst); GGML_UNUSED(src1_ddf_i); From 11474e756de3f56b760986e73086d40e787e52f8 Mon Sep 17 00:00:00 2001 From: Amir Date: Tue, 21 May 2024 17:13:12 +0300 Subject: [PATCH 095/137] examples: cache hf model when --model not provided (#7353) * examples: cache hf model when --model not provided * examples: cache hf model when --model not provided * examples: cache hf model when --model not provided * examples: cache hf model when --model not provided * examples: cache hf model when --model not provided --- common/common.cpp | 32 +++++++++++++++++++++++++++++++- common/common.h | 1 + examples/main/README.md | 2 ++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/common/common.cpp b/common/common.cpp index e624fc7f3..ae11650b4 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1354,7 +1354,12 @@ void gpt_params_handle_model_default(gpt_params & params) { } params.hf_file = params.model; } else if (params.model.empty()) { - params.model = "models/" + string_split(params.hf_file, '/').back(); + std::string cache_directory = get_cache_directory(); + const bool success = create_directory_with_parents(cache_directory); + if (!success) { + throw std::runtime_error("failed to create cache directory: " + cache_directory); + } + params.model = cache_directory + string_split(params.hf_file, '/').back(); } } else if (!params.model_url.empty()) { if (params.model.empty()) { @@ -2516,6 +2521,31 @@ bool create_directory_with_parents(const std::string & path) { #endif // _WIN32 } +std::string get_cache_directory() { + std::string cache_directory = ""; + if (getenv("LLAMA_CACHE")) { + cache_directory = std::getenv("LLAMA_CACHE"); + if (cache_directory.back() != DIRECTORY_SEPARATOR) { + cache_directory += DIRECTORY_SEPARATOR; + } + } else { +#ifdef __linux__ + if (std::getenv("XDG_CACHE_HOME")) { + cache_directory = std::getenv("XDG_CACHE_HOME"); + } else { + cache_directory = std::getenv("HOME") + std::string("/.cache/"); + } +#elif defined(__APPLE__) + cache_directory = std::getenv("HOME") + std::string("/Library/Caches/"); +#elif defined(_WIN32) + cache_directory = std::getenv("APPDATA"); +#endif // __linux__ + cache_directory += "llama.cpp"; + cache_directory += DIRECTORY_SEPARATOR; + } + return cache_directory; +} + void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector & data) { if (data.empty()) { fprintf(stream, "%s:\n", prop_name); diff --git a/common/common.h b/common/common.h index 566490e2f..a8e5e50e6 100644 --- a/common/common.h +++ b/common/common.h @@ -281,6 +281,7 @@ bool llama_should_add_bos_token(const llama_model * model); // bool create_directory_with_parents(const std::string & path); +std::string get_cache_directory(); void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector & data); void dump_vector_int_yaml(FILE * stream, const char * prop_name, const std::vector & data); void dump_string_yaml_multiline(FILE * stream, const char * prop_name, const char * data); diff --git a/examples/main/README.md b/examples/main/README.md index 97e2ae4c2..ee930f4e7 100644 --- a/examples/main/README.md +++ b/examples/main/README.md @@ -325,3 +325,5 @@ These options provide extra functionality and customization when running the LLa - `-ts SPLIT, --tensor-split SPLIT`: When using multiple GPUs this option controls how large tensors should be split across all GPUs. `SPLIT` is a comma-separated list of non-negative values that assigns the proportion of data that each GPU should get in order. For example, "3,2" will assign 60% of the data to GPU 0 and 40% to GPU 1. By default the data is split in proportion to VRAM but this may not be optimal for performance. - `--lora FNAME`: Apply a LoRA (Low-Rank Adaptation) adapter to the model (implies --no-mmap). This allows you to adapt the pretrained model to specific tasks or domains. - `--lora-base FNAME`: Optional model to use as a base for the layers modified by the LoRA adapter. This flag is used in conjunction with the `--lora` flag, and specifies the base model for the adaptation. + +- `-hfr URL --hf-repo URL`: The url to the Hugging Face model repository. Used in conjunction with `--hf-file` or `-hff`. The model is downloaded and stored in the file provided by `-m` or `--model`. If `-m` is not provided, the model is auto-stored in the path specified by the `LLAMA_CACHE` environment variable or in an OS-specific local cache. From c3f8d583560b4f261fa21c976793e538c60cd66c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 21 May 2024 19:53:48 +0300 Subject: [PATCH 096/137] tests : test-tokenizer-0.sh print more info (#7402) --- convert-hf-to-gguf-update.py | 2 +- convert-hf-to-gguf.py | 2 +- tests/test-tokenizer-0.sh | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py index 45404b32b..1923b88ba 100755 --- a/convert-hf-to-gguf-update.py +++ b/convert-hf-to-gguf-update.py @@ -72,7 +72,7 @@ models = [ {"name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", }, {"name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", }, {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", }, - {"name": "stablelm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b", }, + {"name": "stablelm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b", }, {"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", }, {"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", }, {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", }, diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 1acf45bf2..6357d4034 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -447,7 +447,7 @@ class Model: # ref: https://huggingface.co/openai-community/gpt2 res = "gpt-2" if chkhsh == "32d85c31273f8019248f2559fed492d929ea28b17e51d81d3bb36fff23ca72b3": - # ref: https://huggingface.co/stabilityai/stablelm-2-1_6b + # ref: https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b res = "stablelm2" if chkhsh == "6221ad2852e85ce96f791f476e0b390cf9b474c9e3d1362f53a24a06dc8220ff": # ref: https://huggingface.co/smallcloudai/Refact-1_6-base diff --git a/tests/test-tokenizer-0.sh b/tests/test-tokenizer-0.sh index 2fb8632d8..1fec8bbf1 100755 --- a/tests/test-tokenizer-0.sh +++ b/tests/test-tokenizer-0.sh @@ -17,10 +17,15 @@ make -j tests/test-tokenizer-0 printf "Testing %s on %s ...\n" $name $input -python3 ./tests/test-tokenizer-0.py ./models/tokenizers/$name --fname-tok $input > /tmp/test-tokenizer-0-$name-py.log 2>&1 -cat /tmp/test-tokenizer-0-$name-py.log | grep "tokenized in" +set -e +printf "Tokenizing using (py) Python AutoTokenizer ...\n" +python3 ./tests/test-tokenizer-0.py ./models/tokenizers/$name --fname-tok $input > /tmp/test-tokenizer-0-$name-py.log 2>&1 + +printf "Tokenizing using (cpp) llama.cpp ...\n" ./tests/test-tokenizer-0 ./models/ggml-vocab-$name.gguf $input > /tmp/test-tokenizer-0-$name-cpp.log 2>&1 + +cat /tmp/test-tokenizer-0-$name-py.log | grep "tokenized in" cat /tmp/test-tokenizer-0-$name-cpp.log | grep "tokenized in" diff $input.tok $input.tokcpp > /dev/null 2>&1 From fcf6538ba6702c55eaec70da9a75c81d04900a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Tue, 21 May 2024 19:27:12 +0200 Subject: [PATCH 097/137] CUDA: fix unused warning in mmq.cu (#7442) --- ggml-cuda/mmq.cu | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ggml-cuda/mmq.cu b/ggml-cuda/mmq.cu index 5b540d375..933d799ce 100644 --- a/ggml-cuda/mmq.cu +++ b/ggml-cuda/mmq.cu @@ -1220,6 +1220,7 @@ template static __global__ void load_tiles_q4_0, VDR_Q4_0_Q8_1_MMQ, vec_dot_q4_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q4_0_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1244,6 +1245,7 @@ template static __global__ void load_tiles_q4_1, VDR_Q4_1_Q8_1_MMQ, vec_dot_q4_1_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q4_1_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1266,6 +1268,7 @@ template static __global__ void load_tiles_q5_0, VDR_Q5_0_Q8_1_MMQ, vec_dot_q5_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q5_0_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1288,6 +1291,7 @@ mul_mat_q5_1( load_tiles_q5_1, VDR_Q5_1_Q8_1_MMQ, vec_dot_q5_1_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q5_1_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1310,6 +1314,7 @@ template static __global__ void load_tiles_q8_0, VDR_Q8_0_Q8_1_MMQ, vec_dot_q8_0_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q8_0_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1332,6 +1337,7 @@ mul_mat_q2_K( load_tiles_q2_K, VDR_Q2_K_Q8_1_MMQ, vec_dot_q2_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q2_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1356,6 +1362,7 @@ template static __global__ void load_tiles_q3_K, VDR_Q3_K_Q8_1_MMQ, vec_dot_q3_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q3_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1380,6 +1387,7 @@ template static __global__ void load_tiles_q4_K, VDR_Q4_K_Q8_1_MMQ, vec_dot_q4_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q4_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1402,6 +1410,7 @@ mul_mat_q5_K( load_tiles_q5_K, VDR_Q5_K_Q8_1_MMQ, vec_dot_q5_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q5_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A @@ -1426,6 +1435,7 @@ template static __global__ void load_tiles_q6_K, VDR_Q6_K_Q8_1_MMQ, vec_dot_q6_K_q8_1_mul_mat> (vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst); #else + GGML_UNUSED(get_arch_config_device); GGML_UNUSED(vec_dot_q6_K_q8_1_mul_mat); NO_DEVICE_CODE; #endif // __CUDA_ARCH__ >= MIN_CC_DP4A From e402de364b643cb89ea9f43057733b5d36298670 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Tue, 21 May 2024 20:40:00 +0100 Subject: [PATCH 098/137] `grammars`: fix resampling logic regression (#7424) --- common/sampling.cpp | 13 +++++++------ examples/main/main.cpp | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/common/sampling.cpp b/common/sampling.cpp index f0f1b92d3..7fc2e2158 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -179,7 +179,7 @@ static llama_token llama_sampling_sample_impl( struct llama_context * ctx_main, struct llama_context * ctx_cfg, const int idx, - bool is_resampling) { // Add a parameter to indicate if we are resampling + bool is_resampling) { const llama_sampling_params & params = ctx_sampling->params; const float temp = params.temp; @@ -188,8 +188,8 @@ static llama_token llama_sampling_sample_impl( const float mirostat_eta = params.mirostat_eta; std::vector original_logits; - auto cur_p = llama_sampling_prepare(ctx_sampling, ctx_main, ctx_cfg, idx, !is_resampling, &original_logits); - if (!is_resampling) { + auto cur_p = llama_sampling_prepare(ctx_sampling, ctx_main, ctx_cfg, idx, /* apply_grammar= */ is_resampling, &original_logits); + if (ctx_sampling->grammar != NULL && !is_resampling) { GGML_ASSERT(!original_logits.empty()); } llama_token id = 0; @@ -252,7 +252,7 @@ static llama_token llama_sampling_sample_impl( // Restore logits from the copy std::copy(original_logits.begin(), original_logits.end(), logits); - return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, true); // Pass true for is_resampling + return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, /* is_resampling= */ true); } } @@ -285,7 +285,8 @@ static llama_token_data_array llama_sampling_prepare_impl( // Get a pointer to the logits float * logits = llama_get_logits_ith(ctx_main, idx); - if (apply_grammar && original_logits != NULL) { + if (ctx_sampling->grammar != NULL && !apply_grammar) { + GGML_ASSERT(original_logits != NULL); // Only make a copy of the original logits if we are not applying grammar checks, not sure if I actually have to do this. *original_logits = {logits, logits + llama_n_vocab(llama_get_model(ctx_main))}; } @@ -342,7 +343,7 @@ llama_token llama_sampling_sample( struct llama_context * ctx_cfg, const int idx) { // Call the implementation function with is_resampling set to false by default - return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, false); + return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, /* is_resampling= */ false); } llama_token_data_array llama_sampling_prepare( diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 9dee41001..832b51ee0 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -707,7 +707,7 @@ int main(int argc, char ** argv) { const llama_token id = llama_sampling_sample(ctx_sampling, ctx, ctx_guidance); - llama_sampling_accept(ctx_sampling, ctx, id, true); + llama_sampling_accept(ctx_sampling, ctx, id, /* apply_grammar= */ true); LOG("last: %s\n", LOG_TOKENS_TOSTR_PRETTY(ctx, ctx_sampling->prev).c_str()); @@ -728,7 +728,7 @@ int main(int argc, char ** argv) { // push the prompt in the sampling context in order to apply repetition penalties later // for the prompt, we don't apply grammar rules - llama_sampling_accept(ctx_sampling, ctx, embd_inp[n_consumed], false); + llama_sampling_accept(ctx_sampling, ctx, embd_inp[n_consumed], /* apply_grammar= */ false); ++n_consumed; if ((int) embd.size() >= params.n_batch) { From 6369bf04336ab60e5c892dd77a3246df91015147 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 21 May 2024 23:03:42 +0300 Subject: [PATCH 099/137] metal : handle F16 inf values, fix FA partial offload (#7434) ggml-ci --- ggml-metal.metal | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/ggml-metal.metal b/ggml-metal.metal index 386e9195f..cf262e834 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -2204,11 +2204,7 @@ kernel void kernel_flash_attn_ext_f16( // pointer to the mask device const half * mp = (device const half *) (mask + iq1*nb31); - // prepare diagonal scale matrix - simdgroup_float8x8 mscale(scale); - - // prepare diagonal slope matrix - simdgroup_float8x8 mslope(1.0f); + float slope = 1.0f; // ALiBi if (max_bias > 0.0f) { @@ -2217,7 +2213,7 @@ kernel void kernel_flash_attn_ext_f16( const float base = h < n_head_log2 ? m0 : m1; const int exph = h < n_head_log2 ? h + 1 : 2*(h - n_head_log2) + 1; - mslope = simdgroup_float8x8(pow(base, exph)); + slope = pow(base, exph); } // loop over the KV cache @@ -2242,18 +2238,20 @@ kernel void kernel_flash_attn_ext_f16( simdgroup_multiply_accumulate(mqk, mq[i], mk, mqk); } + simdgroup_store(mqk, ss + 8*cc, TF, 0, false); + + const short tx = tiisg%4; + const short ty = tiisg/4; + if (mask != q) { // mqk = mqk*scale + mask*slope - simdgroup_half8x8 mm; - simdgroup_load(mm, mp + ic + 8*cc, nb31/sizeof(half), 0, false); - simdgroup_multiply(mm, mslope, mm); - simdgroup_multiply_accumulate(mqk, mqk, mscale, mm); + ss[8*cc + ty*TF + 2*tx + 0] = scale*ss[8*cc + ty*TF + 2*tx + 0] + slope*mp[ic + 8*cc + ty*nb31/sizeof(half) + 2*tx + 0]; + ss[8*cc + ty*TF + 2*tx + 1] = scale*ss[8*cc + ty*TF + 2*tx + 1] + slope*mp[ic + 8*cc + ty*nb31/sizeof(half) + 2*tx + 1]; } else { // mqk = mqk*scale - simdgroup_multiply(mqk, mscale, mqk); + ss[8*cc + ty*TF + 2*tx + 0] *= scale; + ss[8*cc + ty*TF + 2*tx + 1] *= scale; } - - simdgroup_store(mqk, ss + 8*cc, TF, 0, false); } } @@ -2816,8 +2814,7 @@ kernel void kernel_cpy_f32_f16( for (int64_t i00 = tpitg.x; i00 < ne00; i00 += ntg.x) { device const float * src = (device float *)((device char *) src0 + i03*nb03 + i02*nb02 + i01*nb01 + i00*nb00); - // TODO: is there a better way to handle -INFINITY? - dst_data[i00] = src[0] == -INFINITY ? -MAXHALF : src[0]; + dst_data[i00] = src[0]; } } From 201cc11afa0a1950e1f632390b2ac6c937a0d8f0 Mon Sep 17 00:00:00 2001 From: liuwei-git <14815172+liuwei-git@users.noreply.github.com> Date: Wed, 22 May 2024 04:28:32 +0800 Subject: [PATCH 100/137] llama : add phi3 128K model support (#7225) * add phi3 128k support in convert-hf-to-gguf * add phi3 128k support in cuda * address build warnings on llama.cpp * adjust index value in cuda long rope freq factors * add long rope support in ggml cpu backend * make freq factors only depend on ctx size * remove unused rope scaling type 'su' frin gguf converter * fix flint warnings on convert-hf-to-gguf.py * set to the short freq factor when context size is small than trained context size * add one line of comments * metal : support rope freq_factors * ggml : update ggml_rope_ext API to support freq. factors * backends : add dev messages to support rope freq. factors * minor : style * tests : update to use new rope API * backends : fix pragma semicolons * minor : cleanup * llama : move rope factors from KV header to tensors * llama : remove tmp assert * cuda : fix compile warning * convert : read/write n_head_kv * llama : fix uninitialized tensors --------- Co-authored-by: Georgi Gerganov --- convert-hf-to-gguf.py | 49 +++- examples/finetune/finetune.cpp | 4 +- .../train-text-from-scratch.cpp | 4 +- ggml-cuda/rope.cu | 72 +++-- ggml-kompute.cpp | 4 + ggml-metal.m | 121 ++++---- ggml-metal.metal | 6 +- ggml-sycl.cpp | 3 + ggml-vulkan.cpp | 4 + ggml.c | 88 +++++- ggml.h | 49 +++- gguf-py/gguf/constants.py | 17 +- gguf-py/gguf/gguf_writer.py | 3 + llama.cpp | 277 +++++++++++------- tests/test-backend-ops.cpp | 16 +- 15 files changed, 484 insertions(+), 233 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 6357d4034..daad1c4fc 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -14,6 +14,7 @@ from pathlib import Path from hashlib import sha256 from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterable, Iterator, Sequence, TypeVar, cast +import math import numpy as np import torch @@ -1784,23 +1785,59 @@ class Phi3MiniModel(Model): def set_gguf_parameters(self): block_count = self.find_hparam(["num_hidden_layers", "n_layer"]) - rot_pct = 1.0 n_embd = self.find_hparam(["hidden_size", "n_embd"]) n_head = self.find_hparam(["num_attention_heads", "n_head"]) + n_head_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"]) rms_eps = self.find_hparam(["rms_norm_eps"]) + max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"]) + orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"]) + rope_dims = n_embd // n_head self.gguf_writer.add_name("Phi3") - self.gguf_writer.add_context_length(self.find_hparam(["n_positions", "max_position_embeddings"])) - + self.gguf_writer.add_context_length(max_pos_embds) + self.gguf_writer.add_rope_scaling_orig_ctx_len(orig_max_pos_embds) self.gguf_writer.add_embedding_length(n_embd) - self.gguf_writer.add_feed_forward_length(8192) + self.gguf_writer.add_feed_forward_length(self.find_hparam(["intermediate_size"])) self.gguf_writer.add_block_count(block_count) self.gguf_writer.add_head_count(n_head) - self.gguf_writer.add_head_count_kv(n_head) + self.gguf_writer.add_head_count_kv(n_head_kv) self.gguf_writer.add_layer_norm_rms_eps(rms_eps) - self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head) + self.gguf_writer.add_rope_dimension_count(rope_dims) + self.gguf_writer.add_rope_freq_base(self.find_hparam(["rope_theta"])) self.gguf_writer.add_file_type(self.ftype) + # write rope scaling for long context (128k) model + rope_scaling = self.find_hparam(['rope_scaling'], True) + if (rope_scaling is None): + return + + scale = max_pos_embds / orig_max_pos_embds + + rope_scaling_type = rope_scaling.get('type', '').lower() + if len(rope_scaling_type) == 0: + raise KeyError('Missing the required key rope_scaling.type') + + if rope_scaling_type == 'su': + attn_factor = math.sqrt(1 + math.log(scale) / math.log(orig_max_pos_embds)) if scale > 1.0 else 1.0 + elif rope_scaling_type == 'yarn': + attn_factor = 0.1 * math.log(scale) + 1.0 if scale > 1.0 else 1.0 + else: + raise NotImplementedError(f'The rope scaling type {rope_scaling_type} is not supported yet') + + self.gguf_writer.add_rope_scaling_attn_factors(attn_factor) + + long_factors = rope_scaling.get('long_factor', None) + short_factors = rope_scaling.get('short_factor', None) + + if long_factors is None or short_factors is None: + raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor') + + if len(long_factors) != len(short_factors) or len(long_factors) != rope_dims / 2: + raise ValueError(f'The length of rope long and short factors must be {rope_dims / 2}') + + self.gguf_writer.add_tensor(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.ROPE_FACTORS_LONG] + ".weight", np.array(long_factors, dtype=np.float32)) + self.gguf_writer.add_tensor(gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.ROPE_FACTORS_SHORT] + ".weight", np.array(short_factors, dtype=np.float32)) + @Model.register("PlamoForCausalLM") class PlamoModel(Model): diff --git a/examples/finetune/finetune.cpp b/examples/finetune/finetune.cpp index 22743b1bf..992426c1b 100644 --- a/examples/finetune/finetune.cpp +++ b/examples/finetune/finetune.cpp @@ -563,8 +563,8 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( // not capturing these, to silcence warnings const int rope_mode = 0; - return ggml_rope_custom(ctx, - t, KQ_pos, n_rot, rope_mode, n_ctx, 0, + return ggml_rope_ext(ctx, + t, KQ_pos, nullptr, n_rot, rope_mode, n_ctx, 0, rope_freq_base, rope_freq_scale, 0.0f, 1.0f, 0.0f, 0.0f ); }; diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp index 587418cc7..45bdfa8f5 100644 --- a/examples/train-text-from-scratch/train-text-from-scratch.cpp +++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp @@ -301,8 +301,8 @@ static struct ggml_tensor * llama_build_train_graphs( // not capturing these, to silcence warnings const int rope_mode = 0; - return ggml_rope_custom( - ctx, t, KQ_pos, n_rot, rope_mode, n_ctx, 0, rope_freq_base, rope_freq_scale, 0.0f, 1.0f, 0.0f, 0.0f + return ggml_rope_ext( + ctx, t, KQ_pos, nullptr, n_rot, rope_mode, n_ctx, 0, rope_freq_base, rope_freq_scale, 0.0f, 1.0f, 0.0f, 0.0f ); }; diff --git a/ggml-cuda/rope.cu b/ggml-cuda/rope.cu index 4b0d2e5ad..4a558f4b3 100644 --- a/ggml-cuda/rope.cu +++ b/ggml-cuda/rope.cu @@ -58,10 +58,10 @@ static __global__ void rope( dst[i + 1] = x0*sin_theta + x1*cos_theta; } -template +template static __global__ void rope_neox( const T * x, T * dst, int ncols, int n_dims, const int32_t * pos, float freq_scale, int p_delta_rows, - float ext_factor, float attn_factor, rope_corr_dims corr_dims, float theta_scale, float inv_ndims + float ext_factor, float attn_factor, rope_corr_dims corr_dims, float theta_scale, float inv_ndims, const float * freq_factors ) { const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y); @@ -88,7 +88,9 @@ static __global__ void rope_neox( float cur_rot = inv_ndims * ic - ib; const int p = has_pos ? pos[i2] : 0; - const float theta_base = p*freq_scale*powf(theta_scale, col/2.0f); + const float freq_factor = has_freq_facs ? freq_factors[ic/2] : 1.0f; + + const float theta_base = p*freq_scale*powf(theta_scale, col/2.0f)/freq_factor; float cos_theta, sin_theta; rope_yarn(theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta); @@ -164,7 +166,7 @@ static void rope_cuda( template static void rope_neox_cuda( const T * x, T * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows, - float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream + float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream ) { GGML_ASSERT(ncols % 2 == 0); const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1); @@ -175,15 +177,29 @@ static void rope_neox_cuda( const float inv_ndims = -1.0f / n_dims; if (pos == nullptr) { - rope_neox<<>>( - x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims, - theta_scale, inv_ndims - ); + if (freq_factors == nullptr) { + rope_neox<<>>( + x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims, + theta_scale, inv_ndims, freq_factors + ); + } else { + rope_neox<<>>( + x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims, + theta_scale, inv_ndims, freq_factors + ); + } } else { - rope_neox<<>>( - x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims, - theta_scale, inv_ndims - ); + if (freq_factors == nullptr) { + rope_neox<<>>( + x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims, + theta_scale, inv_ndims, freq_factors + ); + } else { + rope_neox<<>>( + x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims, + theta_scale, inv_ndims, freq_factors + ); + } } } @@ -214,24 +230,27 @@ static void rope_cuda_f32( static void rope_neox_cuda_f16( const half * x, half * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows, - float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream) { + float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream) { - rope_neox_cuda(x, dst, ncols, n_dims, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream); + rope_neox_cuda(x, dst, ncols, n_dims, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, stream); } static void rope_neox_cuda_f32( const float * x, float * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows, - float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream + float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, const float * freq_factors, cudaStream_t stream ) { - rope_neox_cuda(x, dst, ncols, n_dims, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream); + rope_neox_cuda(x, dst, ncols, n_dims, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, freq_factors, stream); } void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; + const ggml_tensor * src2 = dst->src[2]; + const float * src0_d = (const float *)src0->data; const float * src1_d = (const float *)src1->data; + float * dst_d = (float *)dst->data; cudaStream_t stream = ctx.stream(); @@ -241,7 +260,6 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int64_t ne00 = src0->ne[0]; const int64_t ne01 = src0->ne[1]; - const int64_t ne2 = dst->ne[2]; const int64_t nrows = ggml_nrows(src0); //const int n_past = ((int32_t *) dst->op_params)[0]; @@ -259,16 +277,22 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float)); memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float)); + const float * freq_factors = nullptr; const int32_t * pos = nullptr; - if ((mode & 1) == 0) { - GGML_ASSERT(src1->type == GGML_TYPE_I32); - GGML_ASSERT(src1->ne[0] == ne2); - pos = (const int32_t *) src1_d; - } const bool is_neox = mode & 2; const bool is_glm = mode & 4; + if (is_neox) { + pos = (const int32_t *) src1_d; + + if (src2 != nullptr) { + freq_factors = (const float *) src2->data; + } + } else { + GGML_ASSERT(src2 == nullptr && "TODO: freq_factors not implemented for !is_neox"); + } + rope_corr_dims corr_dims; ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims.v); @@ -280,12 +304,12 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { if (src0->type == GGML_TYPE_F32) { rope_neox_cuda_f32( (const float *)src0_d, (float *)dst_d, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor, - attn_factor, corr_dims, stream + attn_factor, corr_dims, freq_factors, stream ); } else if (src0->type == GGML_TYPE_F16) { rope_neox_cuda_f16( (const half *)src0_d, (half *)dst_d, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor, - attn_factor, corr_dims, stream + attn_factor, corr_dims, freq_factors, stream ); } else { GGML_ASSERT(false); diff --git a/ggml-kompute.cpp b/ggml-kompute.cpp index 3f033d58b..6c6058b2a 100644 --- a/ggml-kompute.cpp +++ b/ggml-kompute.cpp @@ -1677,6 +1677,10 @@ static void ggml_vk_graph_compute(struct ggml_kompute_context * ctx, struct ggml } break; case GGML_OP_ROPE: { +#pragma message("TODO: implement phi3 frequency factors support") +#pragma message(" https://github.com/ggerganov/llama.cpp/pull/7225") + GGML_ASSERT(dst->src[2] == nullptr && "phi3 frequency factors not implemented yet"); + GGML_ASSERT(ne10 == ne02); GGML_ASSERT(src0t == dstt); // const int n_past = ((int32_t *) dst->op_params)[0]; diff --git a/ggml-metal.m b/ggml-metal.m index b0b16dbf7..5d5ad20ad 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -927,22 +927,32 @@ static enum ggml_status ggml_metal_graph_compute( const int64_t ne10 = src1 ? src1->ne[0] : 0; const int64_t ne11 = src1 ? src1->ne[1] : 0; const int64_t ne12 = src1 ? src1->ne[2] : 0; - const int64_t ne13 = src1 ? src1->ne[3] : 0; UNUSED(ne13); + const int64_t ne13 = src1 ? src1->ne[3] : 0; const uint64_t nb10 = src1 ? src1->nb[0] : 0; const uint64_t nb11 = src1 ? src1->nb[1] : 0; const uint64_t nb12 = src1 ? src1->nb[2] : 0; - const uint64_t nb13 = src1 ? src1->nb[3] : 0; UNUSED(nb13); + const uint64_t nb13 = src1 ? src1->nb[3] : 0; - const int64_t ne0 = dst ? dst->ne[0] : 0; - const int64_t ne1 = dst ? dst->ne[1] : 0; - const int64_t ne2 = dst ? dst->ne[2] : 0; - const int64_t ne3 = dst ? dst->ne[3] : 0; + const int64_t ne20 = src2 ? src2->ne[0] : 0; + const int64_t ne21 = src2 ? src2->ne[1] : 0; + const int64_t ne22 = src2 ? src2->ne[2] : 0; GGML_UNUSED(ne22); + const int64_t ne23 = src2 ? src2->ne[3] : 0; GGML_UNUSED(ne23); - const uint64_t nb0 = dst ? dst->nb[0] : 0; - const uint64_t nb1 = dst ? dst->nb[1] : 0; - const uint64_t nb2 = dst ? dst->nb[2] : 0; - const uint64_t nb3 = dst ? dst->nb[3] : 0; + const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20); + const uint64_t nb21 = src2 ? src2->nb[1] : 0; + const uint64_t nb22 = src2 ? src2->nb[2] : 0; + const uint64_t nb23 = src2 ? src2->nb[3] : 0; + + const int64_t ne0 = dst ? dst->ne[0] : 0; + const int64_t ne1 = dst ? dst->ne[1] : 0; + const int64_t ne2 = dst ? dst->ne[2] : 0; + const int64_t ne3 = dst ? dst->ne[3] : 0; + + const uint64_t nb0 = dst ? dst->nb[0] : 0; + const uint64_t nb1 = dst ? dst->nb[1] : 0; + const uint64_t nb2 = dst ? dst->nb[2] : 0; + const uint64_t nb3 = dst ? dst->nb[3] : 0; const enum ggml_type src0t = src0 ? src0->type : GGML_TYPE_COUNT; const enum ggml_type src1t = src1 ? src1->type : GGML_TYPE_COUNT; @@ -1785,16 +1795,6 @@ static enum ggml_status ggml_metal_graph_compute( const int n_as = src0->ne[2]; // src2 = ids - const int64_t ne20 = src2->ne[0]; - const int64_t ne21 = src2->ne[1]; - const int64_t ne22 = src2->ne[2]; GGML_UNUSED(ne22); - const int64_t ne23 = src2->ne[3]; GGML_UNUSED(ne23); - - const uint64_t nb20 = src2->nb[0]; GGML_UNUSED(nb20); - const uint64_t nb21 = src2->nb[1]; - const uint64_t nb22 = src2->nb[2]; GGML_UNUSED(nb22); - const uint64_t nb23 = src2->nb[3]; GGML_UNUSED(nb23); - const enum ggml_type src2t = src2->type; GGML_UNUSED(src2t); GGML_ASSERT(src2t == GGML_TYPE_I32); @@ -2244,7 +2244,13 @@ static enum ggml_status ggml_metal_graph_compute( // skip 3, n_ctx, used in GLM RoPE, unimplemented in metal const int n_orig_ctx = ((int32_t *) dst->op_params)[4]; - float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow; + float freq_base; + float freq_scale; + float ext_factor; + float attn_factor; + float beta_fast; + float beta_slow; + memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float)); memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float)); memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float)); @@ -2252,6 +2258,15 @@ static enum ggml_status ggml_metal_graph_compute( memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float)); memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float)); + const bool is_neox = mode & 2; + const bool is_glm = mode & 4; + + GGML_ASSERT(!is_glm && "GLM RoPE not implemented in Metal"); + + if (!is_neox) { + GGML_ASSERT(id_src2 == nil && "TODO: freq_factors not implemented for !is_neox"); + } + id pipeline = nil; switch (src0->type) { @@ -2263,33 +2278,38 @@ static enum ggml_status ggml_metal_graph_compute( [encoder setComputePipelineState:pipeline]; [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0]; [encoder setBuffer:id_src1 offset:offs_src1 atIndex:1]; - [encoder setBuffer:id_dst offset:offs_dst atIndex:2]; - [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:3]; - [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:4]; - [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:5]; - [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:6]; - [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:7]; - [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:8]; - [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:9]; - [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:10]; - [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:11]; - [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:12]; - [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:13]; - [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:14]; - [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:15]; - [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:16]; - [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:17]; - [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:18]; - [encoder setBytes:&n_past length:sizeof( int) atIndex:19]; - [encoder setBytes:&n_dims length:sizeof( int) atIndex:20]; - [encoder setBytes:&mode length:sizeof( int) atIndex:21]; - [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:22]; - [encoder setBytes:&freq_base length:sizeof( float) atIndex:23]; - [encoder setBytes:&freq_scale length:sizeof( float) atIndex:24]; - [encoder setBytes:&ext_factor length:sizeof( float) atIndex:25]; - [encoder setBytes:&attn_factor length:sizeof( float) atIndex:26]; - [encoder setBytes:&beta_fast length:sizeof( float) atIndex:27]; - [encoder setBytes:&beta_slow length:sizeof( float) atIndex:28]; + if (id_src2 != nil) { + [encoder setBuffer:id_src2 offset:offs_src2 atIndex:2]; + } else { + [encoder setBuffer:id_src0 offset:offs_src0 atIndex:2]; + } + [encoder setBuffer:id_dst offset:offs_dst atIndex:3]; + [encoder setBytes:&ne00 length:sizeof( int64_t) atIndex:4]; + [encoder setBytes:&ne01 length:sizeof( int64_t) atIndex:5]; + [encoder setBytes:&ne02 length:sizeof( int64_t) atIndex:6]; + [encoder setBytes:&ne03 length:sizeof( int64_t) atIndex:7]; + [encoder setBytes:&nb00 length:sizeof(uint64_t) atIndex:8]; + [encoder setBytes:&nb01 length:sizeof(uint64_t) atIndex:9]; + [encoder setBytes:&nb02 length:sizeof(uint64_t) atIndex:10]; + [encoder setBytes:&nb03 length:sizeof(uint64_t) atIndex:11]; + [encoder setBytes:&ne0 length:sizeof( int64_t) atIndex:12]; + [encoder setBytes:&ne1 length:sizeof( int64_t) atIndex:13]; + [encoder setBytes:&ne2 length:sizeof( int64_t) atIndex:14]; + [encoder setBytes:&ne3 length:sizeof( int64_t) atIndex:15]; + [encoder setBytes:&nb0 length:sizeof(uint64_t) atIndex:16]; + [encoder setBytes:&nb1 length:sizeof(uint64_t) atIndex:17]; + [encoder setBytes:&nb2 length:sizeof(uint64_t) atIndex:18]; + [encoder setBytes:&nb3 length:sizeof(uint64_t) atIndex:19]; + [encoder setBytes:&n_past length:sizeof( int) atIndex:20]; + [encoder setBytes:&n_dims length:sizeof( int) atIndex:21]; + [encoder setBytes:&mode length:sizeof( int) atIndex:22]; + [encoder setBytes:&n_orig_ctx length:sizeof( int) atIndex:23]; + [encoder setBytes:&freq_base length:sizeof( float) atIndex:24]; + [encoder setBytes:&freq_scale length:sizeof( float) atIndex:25]; + [encoder setBytes:&ext_factor length:sizeof( float) atIndex:26]; + [encoder setBytes:&attn_factor length:sizeof( float) atIndex:27]; + [encoder setBytes:&beta_fast length:sizeof( float) atIndex:28]; + [encoder setBytes:&beta_slow length:sizeof( float) atIndex:29]; [encoder dispatchThreadgroups:MTLSizeMake(ne01, ne02, ne03) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)]; } break; @@ -2535,11 +2555,6 @@ static enum ggml_status ggml_metal_graph_compute( GGML_ASSERT(!src3 || src3->ne[1] >= GGML_PAD(src0->ne[1], 8) && "the Flash-Attention Metal kernel requires the mask to be padded to 8 and at least n_queries big"); - const uint64_t nb20 = src2 ? src2->nb[0] : 0; GGML_UNUSED(nb20); - const uint64_t nb21 = src2 ? src2->nb[1] : 0; - const uint64_t nb22 = src2 ? src2->nb[2] : 0; - const uint64_t nb23 = src2 ? src2->nb[3] : 0; - const int64_t ne30 = src3 ? src3->ne[0] : 0; GGML_UNUSED(ne30); //const int64_t ne31 = src3 ? src3->ne[1] : 0; const int64_t ne32 = src3 ? src3->ne[2] : 0; GGML_UNUSED(ne32); diff --git a/ggml-metal.metal b/ggml-metal.metal index cf262e834..c5eb25280 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -1640,6 +1640,7 @@ static void rope_yarn_corr_dims( typedef void (rope_t)( device const void * src0, device const int32_t * src1, + device const float * src2, device float * dst, constant int64_t & ne00, constant int64_t & ne01, @@ -1675,6 +1676,7 @@ template kernel void kernel_rope( device const void * src0, device const int32_t * src1, + device const float * src2, device float * dst, constant int64_t & ne00, constant int64_t & ne01, @@ -1744,8 +1746,10 @@ kernel void kernel_rope( // simplified from `(ib * n_dims + ic) * inv_ndims` const float cur_rot = inv_ndims*ic - ib; + const float freq_factor = src2 != src0 ? src2[ic/2] : 1.0f; + + const float theta = theta_0 * pow(freq_base, cur_rot) / freq_factor; - const float theta = theta_0 * pow(freq_base, cur_rot); float cos_theta, sin_theta; rope_yarn(theta, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta); diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index eac8f5579..f486b6c0a 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -14454,6 +14454,9 @@ inline void ggml_sycl_op_rope(const ggml_tensor *src0, const ggml_tensor *src1, ggml_tensor *dst, const float *src0_dd, const float *src1_dd, float *dst_dd, const dpct::queue_ptr &main_stream) { +#pragma message("TODO: implement phi3 frequency factors support") +#pragma message(" https://github.com/ggerganov/llama.cpp/pull/7225") + GGML_ASSERT(dst->src[2] == nullptr && "phi3 frequency factors not implemented yet"); GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16); GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16); diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index aff451b63..16287a280 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -4238,6 +4238,10 @@ static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, } static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +#pragma message("TODO: implement phi3 frequency factors support") +#pragma message(" https://github.com/ggerganov/llama.cpp/pull/7225") + GGML_ASSERT(dst->src[2] == nullptr && "phi3 frequency factors not implemented yet"); + const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; // const int n_ctx = ((int32_t *) dst->op_params)[3]; diff --git a/ggml.c b/ggml.c index 4bd911528..37b16b7a9 100644 --- a/ggml.c +++ b/ggml.c @@ -6231,6 +6231,7 @@ static struct ggml_tensor * ggml_rope_impl( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, + struct ggml_tensor * c, int n_dims, int mode, int n_ctx, @@ -6248,6 +6249,11 @@ static struct ggml_tensor * ggml_rope_impl( GGML_ASSERT(b->type == GGML_TYPE_I32); GGML_ASSERT(a->ne[2] == b->ne[0]); + if (c) { + GGML_ASSERT(c->type == GGML_TYPE_F32); + GGML_ASSERT(c->ne[0] >= n_dims / 2); + } + bool is_node = false; if (a->grad) { @@ -6271,6 +6277,7 @@ static struct ggml_tensor * ggml_rope_impl( result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; result->src[0] = a; result->src[1] = b; + result->src[2] = c; return result; } @@ -6283,7 +6290,7 @@ struct ggml_tensor * ggml_rope( int mode, int n_ctx) { return ggml_rope_impl( - ctx, a, b, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, false + ctx, a, b, NULL, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, false ); } @@ -6295,7 +6302,49 @@ struct ggml_tensor * ggml_rope_inplace( int mode, int n_ctx) { return ggml_rope_impl( - ctx, a, b, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, true + ctx, a, b, NULL, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, false, true + ); +} + +struct ggml_tensor * ggml_rope_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx, + int n_orig_ctx, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow) { + return ggml_rope_impl( + ctx, a, b, c, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, false + ); +} + +struct ggml_tensor * ggml_rope_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx, + int n_orig_ctx, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow) { + return ggml_rope_impl( + ctx, a, b, c, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, true ); } @@ -6314,7 +6363,7 @@ struct ggml_tensor * ggml_rope_custom( float beta_fast, float beta_slow) { return ggml_rope_impl( - ctx, a, b, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale, + ctx, a, b, NULL, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, false ); } @@ -6334,27 +6383,18 @@ struct ggml_tensor * ggml_rope_custom_inplace( float beta_fast, float beta_slow) { return ggml_rope_impl( - ctx, a, b, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale, + ctx, a, b, NULL, n_dims, mode, n_ctx, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow, 0.0f, false, true ); } -struct ggml_tensor * ggml_rope_xpos_inplace( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b, - int n_dims, - float base, - bool down) { - return ggml_rope_impl(ctx, a, b, n_dims, 0, 0, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, base, down, true); -} - // ggml_rope_back struct ggml_tensor * ggml_rope_back( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, + struct ggml_tensor * c, int n_dims, int mode, int n_ctx, @@ -6370,6 +6410,7 @@ struct ggml_tensor * ggml_rope_back( GGML_ASSERT(ggml_is_vector(b)); GGML_ASSERT(b->type == GGML_TYPE_I32); GGML_ASSERT(a->ne[2] == b->ne[0]); + GGML_ASSERT(c == NULL && "freq factors not implemented yet"); GGML_ASSERT((mode & 4) == 0 && "ggml_rope_back() for ChatGLM not implemented yet"); @@ -14304,6 +14345,7 @@ static void ggml_compute_forward_rope_f32( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; + const struct ggml_tensor * src2 = dst->src[2]; if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; @@ -14363,6 +14405,17 @@ static void ggml_compute_forward_rope_f32( const bool is_neox = mode & 2; const bool is_glm = mode & 4; + const float * freq_factors = NULL; + if (is_neox) { + if (src2 != NULL) { + GGML_ASSERT(src2->type == GGML_TYPE_F32); + GGML_ASSERT(src2->ne[0] >= n_dims / 2); + freq_factors = (const float *) src2->data; + } + } else { + GGML_ASSERT(src2 == NULL && "TODO: freq_factors not implemented for mode 1"); + } + // backward process uses inverse rotation by cos and sin. // cos and sin build a rotation matrix, where the inverse is the transpose. // this essentially just switches the sign of sin. @@ -14439,10 +14492,11 @@ static void ggml_compute_forward_rope_f32( // simplified from `(ib * n_dims + ic) * inv_ndims` float cur_rot = inv_ndims * ic - ib; + float freq_factor = freq_factors ? freq_factors[ic/2] : 1.0f; float cos_theta, sin_theta; rope_yarn( - theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, + theta_base/freq_factor, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta ); sin_theta *= sin_sign; @@ -18387,6 +18441,7 @@ static struct ggml_tensor * ggml_sub_or_set(struct ggml_context * ctx, struct gg static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor * tensor, struct ggml_hash_set zero_table) { struct ggml_tensor * src0 = tensor->src[0]; struct ggml_tensor * src1 = tensor->src[1]; + struct ggml_tensor * src2 = tensor->src[2]; switch (tensor->op) { case GGML_OP_DUP: @@ -18918,6 +18973,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor ggml_rope_back(ctx, tensor->grad, src1, + src2, n_dims, mode, n_ctx, @@ -18957,6 +19013,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor ggml_rope_impl(ctx, tensor->grad, src1, + src2, n_dims, mode, n_ctx, @@ -19038,7 +19095,6 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor masked); } - struct ggml_tensor * src2 = tensor->src[2]; const int64_t elem_q = ggml_nelements(src0); const int64_t elem_k = ggml_nelements(src1); const int64_t elem_v = ggml_nelements(src2); diff --git a/ggml.h b/ggml.h index 774757101..35ac9110c 100644 --- a/ggml.h +++ b/ggml.h @@ -1465,6 +1465,7 @@ extern "C" { // if mode & 4 == 1, ChatGLM style // // b is an int32 vector with size a->ne[2], it contains the positions + // c is freq factors (e.g. phi3-128k), (optional) GGML_API struct ggml_tensor * ggml_rope( struct ggml_context * ctx, struct ggml_tensor * a, @@ -1483,10 +1484,11 @@ extern "C" { int n_ctx); // custom RoPE - GGML_API struct ggml_tensor * ggml_rope_custom( + GGML_API struct ggml_tensor * ggml_rope_ext( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, + struct ggml_tensor * c, int n_dims, int mode, int n_ctx, @@ -1499,7 +1501,23 @@ extern "C" { float beta_slow); // in-place, returns view(a) - GGML_API struct ggml_tensor * ggml_rope_custom_inplace( + GGML_API struct ggml_tensor * ggml_rope_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx, + int n_orig_ctx, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, @@ -1512,20 +1530,28 @@ extern "C" { float ext_factor, float attn_factor, float beta_fast, - float beta_slow); + float beta_slow), + "use ggml_rope_ext instead"); - // compute correction dims for YaRN RoPE scaling - GGML_CALL void ggml_rope_yarn_corr_dims( - int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]); - - // xPos RoPE, in-place, returns view(a) - GGML_API struct ggml_tensor * ggml_rope_xpos_inplace( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace( struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, int n_dims, - float base, - bool down); + int mode, + int n_ctx, + int n_orig_ctx, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext_inplace instead"); + + // compute correction dims for YaRN RoPE scaling + GGML_CALL void ggml_rope_yarn_corr_dims( + int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]); // rotary position embedding backward, i.e compute dx from dy // a - dy @@ -1533,6 +1559,7 @@ extern "C" { struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, + struct ggml_tensor * c, int n_dims, int mode, int n_ctx, diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 692120f4d..42df2e4d0 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -57,12 +57,13 @@ class Keys: CAUSAL = "{arch}.attention.causal" class Rope: - DIMENSION_COUNT = "{arch}.rope.dimension_count" - FREQ_BASE = "{arch}.rope.freq_base" - SCALING_TYPE = "{arch}.rope.scaling.type" - SCALING_FACTOR = "{arch}.rope.scaling.factor" - SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length" - SCALING_FINETUNED = "{arch}.rope.scaling.finetuned" + DIMENSION_COUNT = "{arch}.rope.dimension_count" + FREQ_BASE = "{arch}.rope.freq_base" + SCALING_TYPE = "{arch}.rope.scaling.type" + SCALING_FACTOR = "{arch}.rope.scaling.factor" + SCALING_ATTN_FACTOR = "{arch}.rope.scaling.attn_factor" + SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length" + SCALING_FINETUNED = "{arch}.rope.scaling.finetuned" class SSM: CONV_KERNEL = "{arch}.ssm.conv_kernel" @@ -148,6 +149,8 @@ class MODEL_TENSOR(IntEnum): OUTPUT = auto() OUTPUT_NORM = auto() ROPE_FREQS = auto() + ROPE_FACTORS_LONG = auto() + ROPE_FACTORS_SHORT = auto() ATTN_Q = auto() ATTN_K = auto() ATTN_V = auto() @@ -225,6 +228,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.OUTPUT_NORM: "output_norm", MODEL_TENSOR.OUTPUT: "output", MODEL_TENSOR.ROPE_FREQS: "rope_freqs", + MODEL_TENSOR.ROPE_FACTORS_LONG: "rope_factors_long", + MODEL_TENSOR.ROPE_FACTORS_SHORT: "rope_factors_short", MODEL_TENSOR.ATTN_NORM: "blk.{bid}.attn_norm", MODEL_TENSOR.ATTN_NORM_2: "blk.{bid}.attn_norm_2", MODEL_TENSOR.ATTN_QKV: "blk.{bid}.attn_qkv", diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index d5e323a52..8b41b54ea 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -433,6 +433,9 @@ class GGUFWriter: def add_rope_scaling_factor(self, value: float) -> None: self.add_float32(Keys.Rope.SCALING_FACTOR.format(arch=self.arch), value) + def add_rope_scaling_attn_factors(self, value: Sequence[float]) -> None: + self.add_float32(Keys.Rope.SCALING_ATTN_FACTOR.format(arch=self.arch), value) + def add_rope_scaling_orig_ctx_len(self, value: int) -> None: self.add_uint32(Keys.Rope.SCALING_ORIG_CTX_LEN.format(arch=self.arch), value) diff --git a/llama.cpp b/llama.cpp index d26fe559a..abff8c1c0 100644 --- a/llama.cpp +++ b/llama.cpp @@ -304,6 +304,7 @@ enum llm_kv { LLM_KV_ROPE_SCALE_LINEAR, LLM_KV_ROPE_SCALING_TYPE, LLM_KV_ROPE_SCALING_FACTOR, + LLM_KV_ROPE_SCALING_ATTN_FACTOR, LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, LLM_KV_ROPE_SCALING_FINETUNED, @@ -381,6 +382,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ROPE_SCALE_LINEAR, "%s.rope.scale_linear" }, { LLM_KV_ROPE_SCALING_TYPE, "%s.rope.scaling.type" }, { LLM_KV_ROPE_SCALING_FACTOR, "%s.rope.scaling.factor" }, + { LLM_KV_ROPE_SCALING_ATTN_FACTOR, "%s.rope.scaling.attn_factor" }, { LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, "%s.rope.scaling.original_context_length" }, { LLM_KV_ROPE_SCALING_FINETUNED, "%s.rope.scaling.finetuned" }, @@ -436,6 +438,8 @@ enum llm_tensor { LLM_TENSOR_OUTPUT, LLM_TENSOR_OUTPUT_NORM, LLM_TENSOR_ROPE_FREQS, + LLM_TENSOR_ROPE_FACTORS_LONG, + LLM_TENSOR_ROPE_FACTORS_SHORT, LLM_TENSOR_ATTN_Q, LLM_TENSOR_ATTN_K, LLM_TENSOR_ATTN_V, @@ -803,18 +807,20 @@ static const std::map> LLM_TENSOR_NA { LLM_ARCH_PHI3, { - { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, - { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, - { LLM_TENSOR_OUTPUT, "output" }, - { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, - { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" }, - { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, - { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, - { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, - { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, - { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, - { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, - { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ROPE_FACTORS_LONG, "rope_factors_long" }, + { LLM_TENSOR_ROPE_FACTORS_SHORT, "rope_factors_short" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, { @@ -1750,6 +1756,7 @@ struct llama_hparams { float f_norm_eps; float f_norm_rms_eps; + float rope_attn_factor = 1.0f; float rope_freq_base_train; float rope_freq_scale_train; uint32_t n_yarn_orig_ctx; @@ -1798,6 +1805,7 @@ struct llama_hparams { if (!is_float_close(this->f_norm_eps, other.f_norm_eps, EPSILON)) return true; if (!is_float_close(this->f_norm_rms_eps, other.f_norm_rms_eps, EPSILON)) return true; + if (!is_float_close(this->rope_attn_factor, other.rope_attn_factor, EPSILON)) return true; if (!is_float_close(this->rope_freq_base_train, other.rope_freq_base_train, EPSILON)) return true; if (!is_float_close(this->rope_freq_scale_train, other.rope_freq_scale_train, EPSILON)) return true; @@ -2103,6 +2111,10 @@ struct llama_model { struct ggml_tensor * output; struct ggml_tensor * output_b; + // long rope factors + struct ggml_tensor * rope_long = nullptr; + struct ggml_tensor * rope_short = nullptr; + std::vector layers; llama_split_mode split_mode; @@ -3306,6 +3318,39 @@ struct llama_model_loader { return get_arr_n(llm_kv(kid), result, required); } + template + bool get_arr(const std::string & key, std::vector & result, const bool required = true) { + const int kid = gguf_find_key(meta, key.c_str()); + + if (kid < 0) { + if (required) { + throw std::runtime_error(format("key not found in model: %s", key.c_str())); + } + return false; + } + + struct GGUFMeta::ArrayInfo arr_info = + GGUFMeta::GKV::get_kv(meta, kid); + + if (arr_info.gt != GGUF_TYPE_FLOAT32 && arr_info.gt != GGUF_TYPE_INT32) { + throw std::runtime_error(format("%s is not a float32 or int32 array", key.c_str())); + } + + // GGML_ASSERT(gguf_type_size(arr_info.gt) == sizeof(T)); + GGML_ASSERT((arr_info.gt != GGUF_TYPE_FLOAT32 || std::is_same::value)); + GGML_ASSERT((arr_info.gt != GGUF_TYPE_INT32 || std::is_same::value)); + + result.resize(arr_info.length); + result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length); + + return true; + } + + template + bool get_arr(const enum llm_kv kid, T& result, const bool required = true) { + return get_arr(llm_kv(kid), result, required); + } + template bool get_key(const std::string & key, T & result, const bool required = true) { auto it = kv_overrides.find(key); @@ -3849,6 +3894,8 @@ static void llm_load_hparams( } hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale; + ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false); + // sanity check for n_rot (optional) { hparams.n_rot = (hparams.n_head == 0) ? 0 : hparams.n_embd / hparams.n_head; @@ -4880,6 +4927,7 @@ static bool llm_load_tensors( // create tensors for the weights { const int64_t n_embd = hparams.n_embd; + const int64_t n_embd_head = n_embd / hparams.n_head; const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa(); const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa(); const int64_t n_embd_gqa = n_embd_v_gqa; @@ -5591,6 +5639,9 @@ static bool llm_load_tensors( { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }); + model.rope_long = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight"), { n_embd_head/2 }, false); + model.rope_short = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight"), { n_embd_head/2 }, false); + // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }); @@ -5601,12 +5652,12 @@ static bool llm_load_tensors( ggml_context* ctx_layer = ctx_for_layer(i); ggml_context* ctx_split = ctx_for_layer_split(i); - auto& layer = model.layers[i]; + auto & layer = model.layers[i]; layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }); layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, n_embd + 2 * n_embd_gqa }, false); - layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd }); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd }); layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }); @@ -6821,17 +6872,20 @@ struct llm_build_context { cb(lctx.inp_K_shift, "K_shift", -1); ggml_set_input(lctx.inp_K_shift); + struct ggml_tensor * rope_factors = build_rope_factors(); + for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * tmp = // we rotate only the first n_rot dimensions - ggml_rope_custom_inplace(ctx0, + ggml_rope_ext_inplace(ctx0, ggml_view_3d(ctx0, kv_self.k_l[il], n_embd_head_k, n_head_kv, n_ctx, ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k), ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa), 0), - lctx.inp_K_shift, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + lctx.inp_K_shift, rope_factors, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + cb(tmp, "K_shifted", il); ggml_build_forward_expand(gf, tmp); } @@ -6934,6 +6988,17 @@ struct llm_build_context { return lctx.inp_pos; } + struct ggml_tensor * build_rope_factors() { + // choose long/short freq factors based on the context size + const auto n_ctx_pre_seq = cparams.n_ctx / cparams.n_seq_max; + + if (n_ctx_pre_seq > hparams.n_yarn_orig_ctx) { + return model.rope_long; + } + + return model.rope_short; + } + struct ggml_tensor * build_inp_out_ids() { lctx.inp_out_ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_outputs); cb(lctx.inp_out_ids, "inp_out_ids", -1); @@ -7041,15 +7106,15 @@ struct llm_build_context { cb(Vcur, "Vcur", il); } - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -7171,13 +7236,13 @@ struct llm_build_context { switch (model.type) { case MODEL_7B: - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -7283,15 +7348,15 @@ struct llm_build_context { struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -7404,14 +7469,14 @@ struct llm_build_context { Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); // using mode = 2 for neox mode - Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -7527,15 +7592,15 @@ struct llm_build_context { cb(Vcur, "Vcur", il); } - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -7679,15 +7744,15 @@ struct llm_build_context { cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -8032,15 +8097,15 @@ struct llm_build_context { cb(Kcur, "Kcur", il); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -8472,15 +8537,15 @@ struct llm_build_context { } - Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -8592,14 +8657,14 @@ struct llm_build_context { Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); // using mode = 2 for neox mode - Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -8703,15 +8768,15 @@ struct llm_build_context { Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -8817,15 +8882,15 @@ struct llm_build_context { Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -8969,8 +9034,8 @@ struct llm_build_context { Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); @@ -8980,8 +9045,8 @@ struct llm_build_context { Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head))); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -9052,6 +9117,9 @@ struct llm_build_context { // KQ_mask (mask for 1 head, it will be broadcasted to all heads) struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); + // rope freq factors for 128k context + struct ggml_tensor * rope_factors = build_rope_factors(); + for (int il = 0; il < n_layer; ++il) { auto residual = inpL; @@ -9088,8 +9156,8 @@ struct llm_build_context { Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); - Qcur = ggml_rope_custom( - ctx0, Qcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, rope_factors, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); @@ -9097,8 +9165,8 @@ struct llm_build_context { Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head))); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, Kcur, inp_pos, n_rot, rope_type, 0, n_orig_ctx, + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Kcur, "Kcur", il); @@ -9204,14 +9272,14 @@ struct llm_build_context { struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_rot, n_head, n_tokens), inp_pos, nullptr, n_embd_head, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_rot, n_head_kv, n_tokens), inp_pos, nullptr, n_embd_head, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Kcur, "Kcur", il); @@ -9412,15 +9480,15 @@ struct llm_build_context { cb(tmpk, "tmpk", il); cb(Vcur, "Vcur", il); - struct ggml_tensor * Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, + struct ggml_tensor * Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, tmpq, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - struct ggml_tensor * Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, + struct ggml_tensor * Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, tmpk, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -9528,15 +9596,15 @@ struct llm_build_context { // cb(Vcur, "Vcur", il); // } - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -9645,15 +9713,15 @@ struct llm_build_context { cb(Vcur, "Vcur", il); } - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -9775,15 +9843,15 @@ struct llm_build_context { cb(Vcur, "Vcur", il); } - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -9895,8 +9963,8 @@ struct llm_build_context { struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); cb(Vcur, "Vcur", il); - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head, n_tokens), inp_pos, nullptr, n_embd_head_k, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Qcur, "Qcur", il); @@ -9904,8 +9972,8 @@ struct llm_build_context { Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); cb(Qcur, "Qcur_scaled", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv, n_tokens), inp_pos, nullptr, n_embd_head_k, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); cb(Kcur, "Kcur", il); @@ -10015,15 +10083,15 @@ struct llm_build_context { cb(Vcur, "Vcur", il); } - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -10305,15 +10373,15 @@ struct llm_build_context { cb(Kcur, "Kcur", il); } - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -10436,15 +10504,15 @@ struct llm_build_context { cb(Vcur, "Vcur", il); } - Qcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); cb(Qcur, "Qcur", il); - Kcur = ggml_rope_custom( - ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -15417,6 +15485,7 @@ struct llama_context * llama_new_context_with_model( cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f; } + cparams.yarn_attn_factor *= hparams.rope_attn_factor; cparams.causal_attn = hparams.causal_attn; if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c74e253db..1493a7ca7 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1763,14 +1763,14 @@ struct test_llama : public test_llm { struct ggml_tensor * Kcur = ggml_mul_mat(ctx, wk, cur); struct ggml_tensor * Vcur = ggml_mul_mat(ctx, wv, cur); - Qcur = ggml_rope_custom( - ctx, ggml_reshape_3d(ctx, Qcur, hp.n_embd_head, hp.n_head, hp.n_tokens), inp_pos, + Qcur = ggml_rope_ext( + ctx, ggml_reshape_3d(ctx, Qcur, hp.n_embd_head, hp.n_head, hp.n_tokens), inp_pos, nullptr, hp.n_rot, 0, 0, hp.n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - Kcur = ggml_rope_custom( - ctx, ggml_reshape_3d(ctx, Kcur, hp.n_embd_head, hp.n_head_kv, hp.n_tokens), inp_pos, + Kcur = ggml_rope_ext( + ctx, ggml_reshape_3d(ctx, Kcur, hp.n_embd_head, hp.n_head_kv, hp.n_tokens), inp_pos, nullptr, hp.n_rot, 0, 0, hp.n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); @@ -1889,13 +1889,13 @@ struct test_falcon : public test_llm { Kcur = ggml_reshape_3d(ctx, Kcur, hp.n_embd_head, hp.n_head_kv, hp.n_tokens); // using mode = 2 for neox mode - Qcur = ggml_rope_custom( - ctx, Qcur, inp_pos, hp.n_rot, 2, 0, hp.n_orig_ctx, + Qcur = ggml_rope_ext( + ctx, Qcur, inp_pos, nullptr, hp.n_rot, 2, 0, hp.n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); - Kcur = ggml_rope_custom( - ctx, Kcur, inp_pos, hp.n_rot, 2, 0, hp.n_orig_ctx, + Kcur = ggml_rope_ext( + ctx, Kcur, inp_pos, nullptr, hp.n_rot, 2, 0, hp.n_orig_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow ); From 3e5faa85032ec3106a2ad831bf412be9ff139f47 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 22 May 2024 11:01:35 +0300 Subject: [PATCH 101/137] cuda : fix rope + add tests (#7452) * cuda : fix rope pos data ggml-ci * ggml : drop mode & 1 == 1 support for ggml_rope ggml-ci * ggml : support freq_factors for f16 rope (CPU) ggml-ci * tests : add rope tests using frequency factors ggml-ci --- ggml-cuda/rope.cu | 4 ++-- ggml.c | 20 +++++++++++++++++-- ggml.h | 2 +- tests/test-backend-ops.cpp | 41 ++++++++++++++++++++++++-------------- 4 files changed, 47 insertions(+), 20 deletions(-) diff --git a/ggml-cuda/rope.cu b/ggml-cuda/rope.cu index 4a558f4b3..50f2cf415 100644 --- a/ggml-cuda/rope.cu +++ b/ggml-cuda/rope.cu @@ -283,9 +283,9 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const bool is_neox = mode & 2; const bool is_glm = mode & 4; - if (is_neox) { - pos = (const int32_t *) src1_d; + pos = (const int32_t *) src1_d; + if (is_neox) { if (src2 != nullptr) { freq_factors = (const float *) src2->data; } diff --git a/ggml.c b/ggml.c index 37b16b7a9..d316e3d31 100644 --- a/ggml.c +++ b/ggml.c @@ -6245,6 +6245,8 @@ static struct ggml_tensor * ggml_rope_impl( float xpos_base, bool xpos_down, bool inplace) { + GGML_ASSERT((mode & 1) == 0 && "mode & 1 == 1 is no longer supported"); + GGML_ASSERT(ggml_is_vector(b)); GGML_ASSERT(b->type == GGML_TYPE_I32); GGML_ASSERT(a->ne[2] == b->ne[0]); @@ -14413,7 +14415,7 @@ static void ggml_compute_forward_rope_f32( freq_factors = (const float *) src2->data; } } else { - GGML_ASSERT(src2 == NULL && "TODO: freq_factors not implemented for mode 1"); + GGML_ASSERT(src2 == NULL && "TODO: freq_factors not implemented for !is_neox"); } // backward process uses inverse rotation by cos and sin. @@ -14529,6 +14531,7 @@ static void ggml_compute_forward_rope_f32( } } +// TODO: deduplicate f16/f32 code static void ggml_compute_forward_rope_f16( const struct ggml_compute_params * params, struct ggml_tensor * dst, @@ -14536,6 +14539,7 @@ static void ggml_compute_forward_rope_f16( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; + const struct ggml_tensor * src2 = dst->src[2]; if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) { return; @@ -14588,6 +14592,17 @@ static void ggml_compute_forward_rope_f16( const bool is_neox = mode & 2; const bool is_glm = mode & 4; + const float * freq_factors = NULL; + if (is_neox) { + if (src2 != NULL) { + GGML_ASSERT(src2->type == GGML_TYPE_F32); + GGML_ASSERT(src2->ne[0] >= n_dims / 2); + freq_factors = (const float *) src2->data; + } + } else { + GGML_ASSERT(src2 == NULL && "TODO: freq_factors not implemented for !is_neox"); + } + // backward process uses inverse rotation by cos and sin. // cos and sin build a rotation matrix, where the inverse is the transpose. // this essentially just switches the sign of sin. @@ -14660,10 +14675,11 @@ static void ggml_compute_forward_rope_f16( // simplified from `(ib * n_dims + ic) * inv_ndims` float cur_rot = inv_ndims * ic - ib; + float freq_factor = freq_factors ? freq_factors[ic/2] : 1.0f; float cos_theta, sin_theta; rope_yarn( - theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, + theta_base/freq_factor, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta ); sin_theta *= sin_sign; diff --git a/ggml.h b/ggml.h index 35ac9110c..08835042c 100644 --- a/ggml.h +++ b/ggml.h @@ -1460,7 +1460,7 @@ extern "C" { struct ggml_tensor * b); // rotary position embedding - // if mode & 1 == 1, skip n_past elements (DEPRECATED) + // if mode & 1 == 1, skip n_past elements (NOT SUPPORTED) // if mode & 2 == 1, GPT-NeoX style // if mode & 4 == 1, ChatGLM style // diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 1493a7ca7..de74585da 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1142,20 +1142,22 @@ struct test_rope : public test_case { int n_dims; int mode; int n_ctx; + bool ff; std::string vars() override { - return VARS_TO_STR5(type, ne, n_dims, mode, n_ctx); + return VARS_TO_STR6(type, ne, n_dims, mode, n_ctx, ff); } test_rope(ggml_type type = GGML_TYPE_F32, std::array ne = {10, 10, 10, 1}, - int n_dims = 10, int mode = 0, int n_ctx = 512) - : type(type), ne(ne), n_dims(n_dims), mode(mode), n_ctx(n_ctx) {} + int n_dims = 10, int mode = 0, int n_ctx = 512, bool ff = false) + : type(type), ne(ne), n_dims(n_dims), mode(mode), n_ctx(n_ctx), ff(ff) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data()); ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne[2]); - ggml_tensor * out = ggml_rope(ctx, a, pos, n_dims, mode, n_ctx); + ggml_tensor * freq = ff ? ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_dims/2) : nullptr; + ggml_tensor * out = ggml_rope_ext(ctx, a, pos, freq, n_dims, mode, n_ctx, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); return out; } @@ -1169,7 +1171,12 @@ struct test_rope : public test_case { } ggml_backend_tensor_set(t, data.data(), 0, ne[2] * sizeof(int)); } else { - init_tensor_uniform(t); + if (t->ne[0] == n_dims/2) { + // frequency factors in the range [0.9f, 1.1f] + init_tensor_uniform(t, 0.9f, 1.1f); + } else { + init_tensor_uniform(t); + } } } } @@ -2188,16 +2195,20 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {32, 2, 32, 1}, true, 0.1f, 8.0f)); for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - test_cases.emplace_back(new test_rope(type, {128, 32, 10, 1}, 128, 0, 512)); // llama 7B - test_cases.emplace_back(new test_rope(type, {128, 40, 10, 1}, 128, 0, 512)); // llama 13B - test_cases.emplace_back(new test_rope(type, {128, 52, 10, 1}, 128, 0, 512)); // llama 30B - test_cases.emplace_back(new test_rope(type, {128, 64, 10, 1}, 128, 0, 512)); // llama 65B - test_cases.emplace_back(new test_rope(type, { 64, 1, 10, 1}, 64, 2, 512)); // neox (falcon 7B) - test_cases.emplace_back(new test_rope(type, { 64, 71, 10, 1}, 64, 2, 512)); // neox (falcon 7B) - test_cases.emplace_back(new test_rope(type, { 64, 8, 10, 1}, 64, 2, 512)); // neox (falcon 40B) - test_cases.emplace_back(new test_rope(type, { 64, 128, 10, 1}, 64, 2, 512)); // neox (falcon 40B) - test_cases.emplace_back(new test_rope(type, { 80, 32, 10, 1}, 20, 2, 512)); // neox (stablelm) - test_cases.emplace_back(new test_rope(type, { 80, 32, 10, 1}, 32, 2, 512)); // neox (phi-2) + // TODO: ff not supported yet for !neox + test_cases.emplace_back(new test_rope(type, {128, 32, 10, 1}, 128, 0, 512, false)); // llama 7B + test_cases.emplace_back(new test_rope(type, {128, 40, 10, 1}, 128, 0, 512, false)); // llama 13B + test_cases.emplace_back(new test_rope(type, {128, 52, 10, 1}, 128, 0, 512, false)); // llama 30B + test_cases.emplace_back(new test_rope(type, {128, 64, 10, 1}, 128, 0, 512, false)); // llama 65B + + for (bool ff : {false, true}) { // freq_factors + test_cases.emplace_back(new test_rope(type, { 64, 1, 10, 1}, 64, 2, 512, ff)); // neox (falcon 7B) + test_cases.emplace_back(new test_rope(type, { 64, 71, 10, 1}, 64, 2, 512, ff)); // neox (falcon 7B) + test_cases.emplace_back(new test_rope(type, { 64, 8, 10, 1}, 64, 2, 512, ff)); // neox (falcon 40B) + test_cases.emplace_back(new test_rope(type, { 64, 128, 10, 1}, 64, 2, 512, ff)); // neox (falcon 40B) + test_cases.emplace_back(new test_rope(type, { 80, 32, 10, 1}, 20, 2, 512, ff)); // neox (stablelm) + test_cases.emplace_back(new test_rope(type, { 80, 32, 10, 1}, 32, 2, 512, ff)); // neox (phi-2) + } } test_cases.emplace_back(new test_concat(GGML_TYPE_F32)); From 95fb0aefab568348da159efdd370e064d1b35f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Wed, 22 May 2024 10:24:29 +0200 Subject: [PATCH 102/137] CUDA: remove incorrect precision check (#7454) --- ggml-cuda/fattn-tile-f32.cu | 3 --- 1 file changed, 3 deletions(-) diff --git a/ggml-cuda/fattn-tile-f32.cu b/ggml-cuda/fattn-tile-f32.cu index 130e7cbdb..54db765e2 100644 --- a/ggml-cuda/fattn-tile-f32.cu +++ b/ggml-cuda/fattn-tile-f32.cu @@ -286,9 +286,6 @@ void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_ten const ggml_tensor * KQV = dst; const ggml_tensor * Q = dst->src[0]; - const int32_t precision = KQV->op_params[2]; - GGML_ASSERT(precision == GGML_PREC_DEFAULT); - if (Q->ne[1] <= 16) { constexpr int cols_per_block = 16; constexpr int parallel_blocks = 4; From 9b3d83318931aa98c487baaa977626931d059e6a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 22 May 2024 12:36:37 +0300 Subject: [PATCH 103/137] cuda : fix compile warning (#7454) --- ggml-cuda/fattn-tile-f32.cu | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ggml-cuda/fattn-tile-f32.cu b/ggml-cuda/fattn-tile-f32.cu index 54db765e2..b8b2f69e1 100644 --- a/ggml-cuda/fattn-tile-f32.cu +++ b/ggml-cuda/fattn-tile-f32.cu @@ -283,8 +283,7 @@ void launch_fattn_tile_f32_64_128(ggml_backend_cuda_context & ctx, ggml_tensor * } void ggml_cuda_flash_attn_ext_tile_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { - const ggml_tensor * KQV = dst; - const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * Q = dst->src[0]; if (Q->ne[1] <= 16) { constexpr int cols_per_block = 16; From 03d8900ebe062355e26a562379daee5f17ea099f Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Wed, 22 May 2024 07:08:18 -0400 Subject: [PATCH 104/137] llama : add missing model type names (#7445) --- llama.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llama.cpp b/llama.cpp index abff8c1c0..d8c6f29a5 100644 --- a/llama.cpp +++ b/llama.cpp @@ -3771,14 +3771,17 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { static const char * llama_model_type_name(e_model type) { switch (type) { + case MODEL_17M: return "17M"; case MODEL_22M: return "22M"; case MODEL_33M: return "33M"; case MODEL_109M: return "109M"; case MODEL_137M: return "137M"; + case MODEL_335M: return "335M"; case MODEL_0_5B: return "0.5B"; case MODEL_1B: return "1B"; case MODEL_2B: return "2B"; case MODEL_3B: return "3B"; + case MODEL_4B: return "4B"; case MODEL_7B: return "7B"; case MODEL_8B: return "8B"; case MODEL_12B: return "12B"; From fcda1128bc5f8eb7e1811708fe9d9867b9aec815 Mon Sep 17 00:00:00 2001 From: "k.h.lai" Date: Wed, 22 May 2024 20:53:21 +0800 Subject: [PATCH 105/137] vulkan: add workaround for iterator boundary check to fix clang-cl debug build (#7426) --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9cc60039a..c09d834fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -505,6 +505,12 @@ if (LLAMA_VULKAN) add_compile_definitions(GGML_USE_VULKAN) + # Workaround to the "can't dereference invalidated vector iterator" bug in clang-cl debug build + # Posssibly relevant: https://stackoverflow.com/questions/74748276/visual-studio-no-displays-the-correct-length-of-stdvector + if (MSVC AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + add_compile_definitions(_ITERATOR_DEBUG_LEVEL=0) + endif() + if (LLAMA_VULKAN_CHECK_RESULTS) add_compile_definitions(GGML_VULKAN_CHECK_RESULTS) endif() From b18532a4efeca8796fea8e36195c81cbfd596a4a Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 22 May 2024 16:10:46 +0200 Subject: [PATCH 106/137] phi3 : duplicate rope factors in each layer (#7447) * phi3 : duplicate rope factors in each layer phi3 : set phi-3 model type as 14B model loader : simplify the process for duplicating model tensors llama-bench : remove default pg test * replace bool parameters in llama_model_loader with named flags --- examples/llama-bench/llama-bench.cpp | 2 +- llama.cpp | 178 ++++++++++++--------------- 2 files changed, 83 insertions(+), 97 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 8b965e199..6bb1f70c3 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -195,7 +195,7 @@ static const cmd_params cmd_params_defaults = { /* model */ {"models/7B/ggml-model-q4_0.gguf"}, /* n_prompt */ {512}, /* n_gen */ {128}, - /* n_pg */ {{512, 128}}, + /* n_pg */ {}, /* n_batch */ {2048}, /* n_ubatch */ {512}, /* type_k */ {GGML_TYPE_F16}, diff --git a/llama.cpp b/llama.cpp index d8c6f29a5..34137c7ad 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1940,6 +1940,10 @@ struct llama_layer { // mamba bias struct ggml_tensor * ssm_conv1d_b; struct ggml_tensor * ssm_dt_b; + + // long rope factors + struct ggml_tensor * rope_long = nullptr; + struct ggml_tensor * rope_short = nullptr; }; struct llama_kv_cell { @@ -2111,10 +2115,6 @@ struct llama_model { struct ggml_tensor * output; struct ggml_tensor * output_b; - // long rope factors - struct ggml_tensor * rope_long = nullptr; - struct ggml_tensor * rope_short = nullptr; - std::vector layers; llama_split_mode split_mode; @@ -3425,11 +3425,15 @@ struct llama_model_loader { return get_tensor_meta(get_tensor_name(i)); } - struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, const struct ggml_tensor * cur) { + struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, const struct ggml_tensor * cur, bool duplicated) { struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur); ggml_set_name(tensor, ggml_get_name(cur)); - n_created++; + if (duplicated) { + size_data += ggml_nbytes(cur); + } else { + n_created++; + } return tensor; } @@ -3464,14 +3468,17 @@ struct llama_model_loader { return cur; } - struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector & ne, bool required = true) { - const struct ggml_tensor * cur = check_tensor_dims(name, ne, required); + static const int TENSOR_NOT_REQUIRED = 1; + static const int TENSOR_DUPLICATED = 2; + + struct ggml_tensor * create_tensor(struct ggml_context * ctx, const std::string & name, const std::vector & ne, int flags = 0) { + const struct ggml_tensor * cur = check_tensor_dims(name, ne, !(flags & TENSOR_NOT_REQUIRED)); if (cur == NULL) { return NULL; } - return create_tensor_for(ctx, cur); + return create_tensor_for(ctx, cur, flags & TENSOR_DUPLICATED); } struct ggml_tensor * create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::vector & ne, size_t offset, bool required = true) { @@ -4139,6 +4146,7 @@ static void llm_load_hparams( switch (hparams.n_layer) { case 24: model.type = e_model::MODEL_1B; break; case 32: model.type = e_model::MODEL_3B; break; + case 40: model.type = e_model::MODEL_14B; break; default: model.type = e_model::MODEL_UNKNOWN; } } break; @@ -4965,12 +4973,10 @@ static bool llm_load_tensors( { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); if (model.arch != LLM_ARCH_MINICPM){ - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (model.output == NULL) { - model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } } } @@ -4989,10 +4995,10 @@ static bool llm_load_tensors( layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); // optional bias tensors - layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, false); - layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, false); - layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, false); - layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, false); + layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); @@ -5003,7 +5009,7 @@ static bool llm_load_tensors( } else { layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}); - layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, false); + layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, llama_model_loader::TENSOR_NOT_REQUIRED); if (layer.ffn_gate_exps) { layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}); layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}); @@ -5045,12 +5051,10 @@ static bool llm_load_tensors( // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (model.output == NULL) { - model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } } @@ -5073,7 +5077,7 @@ static bool llm_load_tensors( layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}); - layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, false); + layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, llama_model_loader::TENSOR_NOT_REQUIRED); if (layer.ffn_gate_exps) { layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}); layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}); @@ -5175,11 +5179,9 @@ static bool llm_load_tensors( model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); if (!model.output) { - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPU - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); // needs to be on GPU } } @@ -5192,8 +5194,8 @@ static bool llm_load_tensors( layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, false); - layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, false); + layer.attn_norm_2 = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.attn_norm_2_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); @@ -5211,12 +5213,10 @@ static bool llm_load_tensors( { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); if (!model.output) { // needs to be on GPU - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } } @@ -5314,14 +5314,14 @@ static bool llm_load_tensors( layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}); - layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, false); - layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, false); + layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}); - layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, false); - layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, false); + layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}); @@ -5383,18 +5383,16 @@ static bool llm_load_tensors( case LLM_ARCH_MPT: { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}, false); + model.pos_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}, llama_model_loader::TENSOR_NOT_REQUIRED); // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, false); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); if (!model.output) { - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // needs to be on GPU - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); // needs to be on GPU } } @@ -5405,31 +5403,31 @@ static bool llm_load_tensors( auto & layer = model.layers[i]; layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); - layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, false); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); - layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, false); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); - layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, false); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); - layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, false); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); - layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, false); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); - layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, false); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, false); - layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, false); + layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.attn_q_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); - layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, false); - layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, false); + layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.attn_k_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); // AWQ ScaleActivation layer - layer.ffn_act = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, false); + layer.ffn_act = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED); } } break; case LLM_ARCH_STABLELM: @@ -5458,17 +5456,17 @@ static bool llm_load_tensors( layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); // optional bias tensors, present in Stable LM 2 1.6B - layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, false); - layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, false); - layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, false); + layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); // optional q and k layernorms, present in StableLM 2 12B - layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {hparams.n_embd_head_k, hparams.n_head}, false); - layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {hparams.n_embd_head_k, hparams.n_head_kv}, false); + layer.attn_q_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {hparams.n_embd_head_k, hparams.n_head}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.attn_k_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {hparams.n_embd_head_k, hparams.n_head_kv}, llama_model_loader::TENSOR_NOT_REQUIRED); // optional FFN norm, not present in StableLM 2 12B which uses parallel residual - layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, false); - layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, false); + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED); layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}); layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}); @@ -5511,12 +5509,10 @@ static bool llm_load_tensors( // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (model.output == NULL) { - model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } } @@ -5614,8 +5610,8 @@ static bool llm_load_tensors( layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); - layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, false); - layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, false); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED); if (layer.wqkv == nullptr) { layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); @@ -5642,9 +5638,6 @@ static bool llm_load_tensors( { model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }); - model.rope_long = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight"), { n_embd_head/2 }, false); - model.rope_short = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight"), { n_embd_head/2 }, false); - // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }); @@ -5659,13 +5652,16 @@ static bool llm_load_tensors( layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }); - layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, n_embd + 2 * n_embd_gqa }, false); + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), { n_embd, n_embd + 2 * n_embd_gqa }, llama_model_loader::TENSOR_NOT_REQUIRED); layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd, n_embd }); layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), { n_embd }); layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd }); layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), { n_embd, 2 * n_ff }); + + layer.rope_long = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FACTORS_LONG, "weight"), { n_embd_head/2 }, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0)); + layer.rope_short = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FACTORS_SHORT, "weight"), { n_embd_head/2 }, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0)); } } break; case LLM_ARCH_PLAMO: @@ -5834,9 +5830,7 @@ static bool llm_load_tensors( // output model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); // same as tok_embd, duplicated to allow offloading - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); // same as tok_embd, duplicated to allow offloading const int64_t n_ff = hparams.n_ff; const int64_t n_embd_head_k = hparams.n_embd_head_k; @@ -5871,12 +5865,10 @@ static bool llm_load_tensors( model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (model.output == NULL) { - model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } } @@ -5927,12 +5919,10 @@ static bool llm_load_tensors( { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed, duplicated to allow offloading if (model.output == NULL) { - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } } @@ -5993,9 +5983,7 @@ static bool llm_load_tensors( { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); // init output from the input tok embed - model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } for (int i = 0; i < n_layer; ++i) { @@ -6027,12 +6015,10 @@ static bool llm_load_tensors( // output { - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (model.output == NULL) { - model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); - ml.n_created--; // artificial tensor - ml.size_data += ggml_nbytes(model.output); + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } } @@ -6875,9 +6861,9 @@ struct llm_build_context { cb(lctx.inp_K_shift, "K_shift", -1); ggml_set_input(lctx.inp_K_shift); - struct ggml_tensor * rope_factors = build_rope_factors(); for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * rope_factors = build_rope_factors(il); struct ggml_tensor * tmp = // we rotate only the first n_rot dimensions ggml_rope_ext_inplace(ctx0, @@ -6991,15 +6977,15 @@ struct llm_build_context { return lctx.inp_pos; } - struct ggml_tensor * build_rope_factors() { + struct ggml_tensor * build_rope_factors(int il) { // choose long/short freq factors based on the context size const auto n_ctx_pre_seq = cparams.n_ctx / cparams.n_seq_max; if (n_ctx_pre_seq > hparams.n_yarn_orig_ctx) { - return model.rope_long; + return model.layers[il].rope_long; } - return model.rope_short; + return model.layers[il].rope_short; } struct ggml_tensor * build_inp_out_ids() { @@ -9120,14 +9106,14 @@ struct llm_build_context { // KQ_mask (mask for 1 head, it will be broadcasted to all heads) struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); - // rope freq factors for 128k context - struct ggml_tensor * rope_factors = build_rope_factors(); - for (int il = 0; il < n_layer; ++il) { auto residual = inpL; // self-attention { + // rope freq factors for 128k context + struct ggml_tensor * rope_factors = build_rope_factors(il); + struct ggml_tensor* attn_norm_output = llm_build_norm(ctx0, inpL, hparams, model.layers[il].attn_norm, NULL, From 38c03478a37e460ecd3a21155b338a83bfed7f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Wed, 22 May 2024 17:58:25 +0200 Subject: [PATCH 107/137] CUDA: fix FA out-of-bounds writes (#7465) --- ggml-cuda/fattn-tile-f16.cu | 4 ++++ ggml-cuda/fattn-tile-f32.cu | 4 ++++ ggml-cuda/fattn-vec-f16.cu | 6 +++++- ggml-cuda/fattn-vec-f32.cu | 6 +++++- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ggml-cuda/fattn-tile-f16.cu b/ggml-cuda/fattn-tile-f16.cu index 4a07ac6ad..586d469c0 100644 --- a/ggml-cuda/fattn-tile-f16.cu +++ b/ggml-cuda/fattn-tile-f16.cu @@ -238,6 +238,10 @@ static __global__ void flash_attn_tile_ext_f16( for (int j_VKQ_0 = 0; j_VKQ_0 < ncols; j_VKQ_0 += nwarps) { const int j_VKQ = j_VKQ_0 + threadIdx.y; + if (ic0 + j_VKQ >= ne01) { + return; + } + half kqsum_j = __low2half(kqsum[j_VKQ_0/nwarps]) + __high2half(kqsum[j_VKQ_0/nwarps]); kqsum_j = warp_reduce_sum(kqsum_j); diff --git a/ggml-cuda/fattn-tile-f32.cu b/ggml-cuda/fattn-tile-f32.cu index b8b2f69e1..b6ef8eb48 100644 --- a/ggml-cuda/fattn-tile-f32.cu +++ b/ggml-cuda/fattn-tile-f32.cu @@ -237,6 +237,10 @@ static __global__ void flash_attn_tile_ext_f32( for (int j_VKQ_0 = 0; j_VKQ_0 < ncols; j_VKQ_0 += nwarps) { const int j_VKQ = j_VKQ_0 + threadIdx.y; + if (ic0 + j_VKQ >= ne01) { + return; + } + float kqsum_j = kqsum[j_VKQ_0/nwarps]; kqsum_j = warp_reduce_sum(kqsum_j); diff --git a/ggml-cuda/fattn-vec-f16.cu b/ggml-cuda/fattn-vec-f16.cu index 54e1ac5d1..7352dcabf 100644 --- a/ggml-cuda/fattn-vec-f16.cu +++ b/ggml-cuda/fattn-vec-f16.cu @@ -212,6 +212,10 @@ static __global__ void flash_attn_vec_ext_f16( #pragma unroll for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) { + if (ic0 + j_VKQ >= ne01) { + break; + } + kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x]; kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]); @@ -223,7 +227,7 @@ static __global__ void flash_attn_vec_ext_f16( dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; } - if (parallel_blocks != 1 && tid < ncols) { + if (parallel_blocks != 1 && tid < ncols && ic0 + tid < ne01) { dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); } #else diff --git a/ggml-cuda/fattn-vec-f32.cu b/ggml-cuda/fattn-vec-f32.cu index 5bcabd092..11476a6c0 100644 --- a/ggml-cuda/fattn-vec-f32.cu +++ b/ggml-cuda/fattn-vec-f32.cu @@ -200,6 +200,10 @@ static __global__ void flash_attn_vec_ext_f32( #pragma unroll for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) { + if (ic0 + j_VKQ >= ne01) { + break; + } + kqsum[j_VKQ] = kqsum_shared[j_VKQ][threadIdx.x]; kqsum[j_VKQ] = warp_reduce_sum(kqsum[j_VKQ]); @@ -211,7 +215,7 @@ static __global__ void flash_attn_vec_ext_f32( dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; } - if (parallel_blocks != 1 && tid < ncols) { + if (parallel_blocks != 1 && tid < ncols && ic0 + tid < ne01) { dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); } } From 6ff13987ad1a9519bee13dd98b6a21cd98979aab Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 22 May 2024 20:04:20 +0300 Subject: [PATCH 108/137] common : normalize naming style (#7462) * common : normalize naming style ggml-ci * common : match declaration / definition order * zig : try to fix build --- build.zig | 12 +- common/common.cpp | 1338 +++++++++++----------- common/common.h | 90 +- common/sampling.cpp | 83 +- common/sampling.h | 5 + common/train.cpp | 2 +- examples/batched/batched.cpp | 2 +- examples/embedding/embedding.cpp | 4 +- examples/eval-callback/eval-callback.cpp | 4 +- examples/imatrix/imatrix.cpp | 4 +- examples/infill/infill.cpp | 16 +- examples/llama-bench/llama-bench.cpp | 2 +- examples/llava/llava-cli.cpp | 2 +- examples/lookahead/lookahead.cpp | 2 +- examples/lookup/lookup.cpp | 2 +- examples/main/main.cpp | 16 +- examples/parallel/parallel.cpp | 2 +- examples/perplexity/perplexity.cpp | 14 +- examples/quantize/quantize.cpp | 2 +- examples/retrieval/retrieval.cpp | 4 +- examples/server/server.cpp | 10 +- 21 files changed, 823 insertions(+), 793 deletions(-) diff --git a/build.zig b/build.zig index 96783574f..267c976b1 100644 --- a/build.zig +++ b/build.zig @@ -129,14 +129,14 @@ pub fn build(b: *std.build.Builder) !void { const clip = make.obj("clip", "examples/llava/clip.cpp"); const llava = make.obj("llava", "examples/llava/llava.cpp"); - _ = make.exe("main", "examples/main/main.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, sampling, console, grammar_parser }); - _ = make.exe("quantize", "examples/quantize/quantize.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo }); - _ = make.exe("perplexity", "examples/perplexity/perplexity.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo }); - _ = make.exe("embedding", "examples/embedding/embedding.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo }); - _ = make.exe("finetune", "examples/finetune/finetune.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, train }); + _ = make.exe("main", "examples/main/main.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo, console, grammar_parser }); + _ = make.exe("quantize", "examples/quantize/quantize.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo }); + _ = make.exe("perplexity", "examples/perplexity/perplexity.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo }); + _ = make.exe("embedding", "examples/embedding/embedding.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo }); + _ = make.exe("finetune", "examples/finetune/finetune.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo, train }); _ = make.exe("train-text-from-scratch", "examples/train-text-from-scratch/train-text-from-scratch.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, train }); - const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, sampling, grammar_parser, clip, llava }); + const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo, grammar_parser, clip, llava }); if (server.target.isWindows()) { server.linkSystemLibrary("ws2_32"); } diff --git a/common/common.cpp b/common/common.cpp index ae11650b4..7500e08ff 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -73,7 +73,11 @@ using json = nlohmann::ordered_json; -int32_t get_num_physical_cores() { +// +// CPU utils +// + +int32_t cpu_get_num_physical_cores() { #ifdef __linux__ // enumerate the set of thread siblings, num entries is num cores std::unordered_set siblings; @@ -142,9 +146,9 @@ static bool is_running_on_efficiency_core(void) { return core_type == intel_atom; } -static int count_math_cpus(int cpu_count) { +static int cpu_count_math_cpus(int n_cpu) { int result = 0; - for (int cpu = 0; cpu < cpu_count; ++cpu) { + for (int cpu = 0; cpu < n_cpu; ++cpu) { if (pin_cpu(cpu)) { return -1; } @@ -162,16 +166,16 @@ static int count_math_cpus(int cpu_count) { /** * Returns number of CPUs on system that are useful for math. */ -int get_math_cpu_count() { +int32_t cpu_get_num_math() { #if defined(__x86_64__) && defined(__linux__) && !defined(__ANDROID__) - int cpu_count = sysconf(_SC_NPROCESSORS_ONLN); - if (cpu_count < 1) { - return get_num_physical_cores(); + int n_cpu = sysconf(_SC_NPROCESSORS_ONLN); + if (n_cpu < 1) { + return cpu_get_num_physical_cores(); } if (is_hybrid_cpu()) { cpu_set_t affinity; if (!pthread_getaffinity_np(pthread_self(), sizeof(affinity), &affinity)) { - int result = count_math_cpus(cpu_count); + int result = cpu_count_math_cpus(n_cpu); pthread_setaffinity_np(pthread_self(), sizeof(affinity), &affinity); if (result > 0) { return result; @@ -179,108 +183,103 @@ int get_math_cpu_count() { } } #endif - return get_num_physical_cores(); + return cpu_get_num_physical_cores(); } -void process_escapes(std::string & input) { - std::size_t input_len = input.length(); - std::size_t output_idx = 0; +// +// CLI argument parsing +// - for (std::size_t input_idx = 0; input_idx < input_len; ++input_idx) { - if (input[input_idx] == '\\' && input_idx + 1 < input_len) { - switch (input[++input_idx]) { - case 'n': input[output_idx++] = '\n'; break; - case 'r': input[output_idx++] = '\r'; break; - case 't': input[output_idx++] = '\t'; break; - case '\'': input[output_idx++] = '\''; break; - case '\"': input[output_idx++] = '\"'; break; - case '\\': input[output_idx++] = '\\'; break; - case 'x': - // Handle \x12, etc - if (input_idx + 2 < input_len) { - const char x[3] = { input[input_idx + 1], input[input_idx + 2], 0 }; - char *err_p = nullptr; - const long val = std::strtol(x, &err_p, 16); - if (err_p == x + 2) { - input_idx += 2; - input[output_idx++] = char(val); - break; - } - } - // fall through - default: input[output_idx++] = '\\'; - input[output_idx++] = input[input_idx]; break; +void gpt_params_handle_model_default(gpt_params & params) { + if (!params.hf_repo.empty()) { + // short-hand to avoid specifying --hf-file -> default it to --model + if (params.hf_file.empty()) { + if (params.model.empty()) { + throw std::invalid_argument("error: --hf-repo requires either --hf-file or --model\n"); } - } else { - input[output_idx++] = input[input_idx]; + params.hf_file = params.model; + } else if (params.model.empty()) { + std::string cache_directory = fs_get_cache_directory(); + const bool success = fs_create_directory_with_parents(cache_directory); + if (!success) { + throw std::runtime_error("failed to create cache directory: " + cache_directory); + } + params.model = cache_directory + string_split(params.hf_file, '/').back(); + } + } else if (!params.model_url.empty()) { + if (params.model.empty()) { + auto f = string_split(params.model_url, '#').front(); + f = string_split(f, '?').front(); + f = string_split(f, '/').back(); + params.model = "models/" + f; + } + } else if (params.model.empty()) { + params.model = DEFAULT_MODEL_PATH; + } +} + +bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { + bool invalid_param = false; + std::string arg; + const std::string arg_prefix = "--"; + llama_sampling_params & sparams = params.sparams; + + for (int i = 1; i < argc; i++) { + arg = argv[i]; + if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) { + std::replace(arg.begin(), arg.end(), '_', '-'); + } + if (!gpt_params_find_arg(argc, argv, arg, params, i, invalid_param)) { + throw std::invalid_argument("error: unknown argument: " + arg); + } + if (invalid_param) { + throw std::invalid_argument("error: invalid parameter for argument: " + arg); } } - input.resize(output_idx); + if (params.prompt_cache_all && + (params.interactive || params.interactive_first || + params.instruct)) { + + throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); + } + + gpt_params_handle_model_default(params); + + if (params.escape) { + string_process_escapes(params.prompt); + string_process_escapes(params.input_prefix); + string_process_escapes(params.input_suffix); + string_process_escapes(sparams.cfg_negative_prompt); + for (auto & antiprompt : params.antiprompt) { + string_process_escapes(antiprompt); + } + } + + if (!params.kv_overrides.empty()) { + params.kv_overrides.emplace_back(); + params.kv_overrides.back().key[0] = 0; + } + + return true; } bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { bool result = true; try { if (!gpt_params_parse_ex(argc, argv, params)) { - gpt_print_usage(argc, argv, gpt_params()); + gpt_params_print_usage(argc, argv, gpt_params()); exit(0); } } catch (const std::invalid_argument & ex) { fprintf(stderr, "%s\n", ex.what()); - gpt_print_usage(argc, argv, gpt_params()); + gpt_params_print_usage(argc, argv, gpt_params()); exit(1); } return result; } -bool parse_kv_override(const char * data, std::vector & overrides) { - const char * sep = strchr(data, '='); - if (sep == nullptr || sep - data >= 128) { - fprintf(stderr, "%s: malformed KV override '%s'\n", __func__, data); - return false; - } - llama_model_kv_override kvo; - std::strncpy(kvo.key, data, sep - data); - kvo.key[sep - data] = 0; - sep++; - if (strncmp(sep, "int:", 4) == 0) { - sep += 4; - kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT; - kvo.val_i64 = std::atol(sep); - } else if (strncmp(sep, "float:", 6) == 0) { - sep += 6; - kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT; - kvo.val_f64 = std::atof(sep); - } else if (strncmp(sep, "bool:", 5) == 0) { - sep += 5; - kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL; - if (std::strcmp(sep, "true") == 0) { - kvo.val_bool = true; - } else if (std::strcmp(sep, "false") == 0) { - kvo.val_bool = false; - } else { - fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data); - return false; - } - } else if (strncmp(sep, "str:", 4) == 0) { - sep += 4; - kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR; - if (strlen(sep) > 127) { - fprintf(stderr, "%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data); - return false; - } - strncpy(kvo.val_str, sep, 127); - kvo.val_str[127] = '\0'; - } else { - fprintf(stderr, "%s: invalid type for KV override '%s'\n", __func__, data); - return false; - } - overrides.emplace_back(std::move(kvo)); - return true; -} - bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param) { llama_sampling_params & sparams = params.sparams; @@ -546,7 +545,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa return true; } const auto sampler_names = string_split(argv[i], ';'); - sparams.samplers_sequence = sampler_types_from_names(sampler_names, true); + sparams.samplers_sequence = llama_sampling_types_from_names(sampler_names, true); return true; } if (arg == "--sampling-seq") { @@ -554,7 +553,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa invalid_param = true; return true; } - sparams.samplers_sequence = sampler_types_from_chars(argv[i]); + sparams.samplers_sequence = llama_sampling_types_from_chars(argv[i]); return true; } if (arg == "--top-p") { @@ -1240,7 +1239,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa return true; } if (arg == "-h" || arg == "--help") { - gpt_print_usage(argc, argv, gpt_params()); + gpt_params_print_usage(argc, argv, gpt_params()); exit(0); } if (arg == "--version") { @@ -1311,7 +1310,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa invalid_param = true; return true; } - if (!parse_kv_override(argv[i], params.kv_overrides)) { + if (!string_parse_kv_override(argv[i], params.kv_overrides)) { fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]); invalid_param = true; return true; @@ -1345,88 +1344,14 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa return false; } -void gpt_params_handle_model_default(gpt_params & params) { - if (!params.hf_repo.empty()) { - // short-hand to avoid specifying --hf-file -> default it to --model - if (params.hf_file.empty()) { - if (params.model.empty()) { - throw std::invalid_argument("error: --hf-repo requires either --hf-file or --model\n"); - } - params.hf_file = params.model; - } else if (params.model.empty()) { - std::string cache_directory = get_cache_directory(); - const bool success = create_directory_with_parents(cache_directory); - if (!success) { - throw std::runtime_error("failed to create cache directory: " + cache_directory); - } - params.model = cache_directory + string_split(params.hf_file, '/').back(); - } - } else if (!params.model_url.empty()) { - if (params.model.empty()) { - auto f = string_split(params.model_url, '#').front(); - f = string_split(f, '?').front(); - f = string_split(f, '/').back(); - params.model = "models/" + f; - } - } else if (params.model.empty()) { - params.model = DEFAULT_MODEL_PATH; - } -} - -bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) { - bool invalid_param = false; - std::string arg; - const std::string arg_prefix = "--"; - llama_sampling_params & sparams = params.sparams; - - for (int i = 1; i < argc; i++) { - arg = argv[i]; - if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) { - std::replace(arg.begin(), arg.end(), '_', '-'); - } - if (!gpt_params_find_arg(argc, argv, arg, params, i, invalid_param)) { - throw std::invalid_argument("error: unknown argument: " + arg); - } - if (invalid_param) { - throw std::invalid_argument("error: invalid parameter for argument: " + arg); - } - } - - if (params.prompt_cache_all && - (params.interactive || params.interactive_first || - params.instruct)) { - - throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); - } - - gpt_params_handle_model_default(params); - - if (params.escape) { - process_escapes(params.prompt); - process_escapes(params.input_prefix); - process_escapes(params.input_suffix); - process_escapes(sparams.cfg_negative_prompt); - for (auto & antiprompt : params.antiprompt) { - process_escapes(antiprompt); - } - } - - if (!params.kv_overrides.empty()) { - params.kv_overrides.emplace_back(); - params.kv_overrides.back().key[0] = 0; - } - - return true; -} - -void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { +void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { const llama_sampling_params & sparams = params.sparams; std::string sampler_type_chars; std::string sampler_type_names; for (const auto sampler_type : sparams.samplers_sequence) { sampler_type_chars += static_cast(sampler_type); - sampler_type_names += sampler_type_to_name_string(sampler_type) + ";"; + sampler_type_names += llama_sampling_type_to_str(sampler_type) + ";"; } sampler_type_names.pop_back(); @@ -1623,7 +1548,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { #endif // LOG_DISABLE_LOGS } -std::string get_system_info(const gpt_params & params) { +std::string gpt_params_get_system_info(const gpt_params & params) { std::ostringstream os; os << "system_info: n_threads = " << params.n_threads; @@ -1635,7 +1560,52 @@ std::string get_system_info(const gpt_params & params) { return os.str(); } -std::string gpt_random_prompt(std::mt19937 & rng) { +// +// String utils +// + +std::vector string_split(std::string input, char separator) { + std::vector parts; + size_t separator_pos = input.find(separator); + while (separator_pos != std::string::npos) { + std::string part = input.substr(0, separator_pos); + parts.emplace_back(part); + input = input.substr(separator_pos + 1); + separator_pos = input.find(separator); + } + parts.emplace_back(input); + return parts; +} + +std::string string_strip(const std::string & str) { + size_t start = 0; + size_t end = str.size(); + while (start < end && std::isspace(str[start])) { + start++; + } + while (end > start && std::isspace(str[end - 1])) { + end--; + } + return str.substr(start, end - start); +} + +std::string string_get_sortable_timestamp() { + using clock = std::chrono::system_clock; + + const clock::time_point current_time = clock::now(); + const time_t as_time_t = clock::to_time_t(current_time); + char timestamp_no_ns[100]; + std::strftime(timestamp_no_ns, 100, "%Y_%m_%d-%H_%M_%S", std::localtime(&as_time_t)); + + const int64_t ns = std::chrono::duration_cast( + current_time.time_since_epoch() % 1000000000).count(); + char timestamp_ns[11]; + snprintf(timestamp_ns, 11, "%09" PRId64, ns); + + return std::string(timestamp_no_ns) + "." + std::string(timestamp_ns); +} + +std::string string_random_prompt(std::mt19937 & rng) { const int r = rng() % 10; switch (r) { case 0: return "So"; @@ -1653,9 +1623,96 @@ std::string gpt_random_prompt(std::mt19937 & rng) { GGML_UNREACHABLE(); } +void string_process_escapes(std::string & input) { + std::size_t input_len = input.length(); + std::size_t output_idx = 0; + + for (std::size_t input_idx = 0; input_idx < input_len; ++input_idx) { + if (input[input_idx] == '\\' && input_idx + 1 < input_len) { + switch (input[++input_idx]) { + case 'n': input[output_idx++] = '\n'; break; + case 'r': input[output_idx++] = '\r'; break; + case 't': input[output_idx++] = '\t'; break; + case '\'': input[output_idx++] = '\''; break; + case '\"': input[output_idx++] = '\"'; break; + case '\\': input[output_idx++] = '\\'; break; + case 'x': + // Handle \x12, etc + if (input_idx + 2 < input_len) { + const char x[3] = { input[input_idx + 1], input[input_idx + 2], 0 }; + char *err_p = nullptr; + const long val = std::strtol(x, &err_p, 16); + if (err_p == x + 2) { + input_idx += 2; + input[output_idx++] = char(val); + break; + } + } + // fall through + default: input[output_idx++] = '\\'; + input[output_idx++] = input[input_idx]; break; + } + } else { + input[output_idx++] = input[input_idx]; + } + } + + input.resize(output_idx); +} + +bool string_parse_kv_override(const char * data, std::vector & overrides) { + const char * sep = strchr(data, '='); + if (sep == nullptr || sep - data >= 128) { + fprintf(stderr, "%s: malformed KV override '%s'\n", __func__, data); + return false; + } + llama_model_kv_override kvo; + std::strncpy(kvo.key, data, sep - data); + kvo.key[sep - data] = 0; + sep++; + if (strncmp(sep, "int:", 4) == 0) { + sep += 4; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT; + kvo.val_i64 = std::atol(sep); + } else if (strncmp(sep, "float:", 6) == 0) { + sep += 6; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_FLOAT; + kvo.val_f64 = std::atof(sep); + } else if (strncmp(sep, "bool:", 5) == 0) { + sep += 5; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_BOOL; + if (std::strcmp(sep, "true") == 0) { + kvo.val_bool = true; + } else if (std::strcmp(sep, "false") == 0) { + kvo.val_bool = false; + } else { + fprintf(stderr, "%s: invalid boolean value for KV override '%s'\n", __func__, data); + return false; + } + } else if (strncmp(sep, "str:", 4) == 0) { + sep += 4; + kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR; + if (strlen(sep) > 127) { + fprintf(stderr, "%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data); + return false; + } + strncpy(kvo.val_str, sep, 127); + kvo.val_str[127] = '\0'; + } else { + fprintf(stderr, "%s: invalid type for KV override '%s'\n", __func__, data); + return false; + } + overrides.emplace_back(std::move(kvo)); + return true; +} + +// +// Filesystem utils +// + // Validate if a filename is safe to use // To validate a full path, split the path by the OS-specific path separator, and validate each part with this function -bool validate_file_name(const std::string & filename) { +bool fs_validate_filename(const std::string & filename) { if (!filename.length()) { // Empty filename invalid return false; @@ -1724,120 +1781,194 @@ bool validate_file_name(const std::string & filename) { return true; } -// -// String utils -// +// returns true if successful, false otherwise +bool fs_create_directory_with_parents(const std::string & path) { +#ifdef _WIN32 + std::wstring_convert> converter; + std::wstring wpath = converter.from_bytes(path); -std::vector string_split(std::string input, char separator) { - std::vector parts; - size_t separator_pos = input.find(separator); - while (separator_pos != std::string::npos) { - std::string part = input.substr(0, separator_pos); - parts.emplace_back(part); - input = input.substr(separator_pos + 1); - separator_pos = input.find(separator); + // if the path already exists, check whether it's a directory + const DWORD attributes = GetFileAttributesW(wpath.c_str()); + if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) { + return true; } - parts.emplace_back(input); - return parts; -} -std::string string_strip(const std::string & str) { - size_t start = 0; - size_t end = str.size(); - while (start < end && std::isspace(str[start])) { - start++; - } - while (end > start && std::isspace(str[end - 1])) { - end--; - } - return str.substr(start, end - start); -} + size_t pos_slash = 0; -std::vector sampler_types_from_names(const std::vector & names, bool allow_alt_names) { - std::unordered_map sampler_canonical_name_map { - {"top_k", llama_sampler_type::TOP_K}, - {"top_p", llama_sampler_type::TOP_P}, - {"typical_p", llama_sampler_type::TYPICAL_P}, - {"min_p", llama_sampler_type::MIN_P}, - {"tfs_z", llama_sampler_type::TFS_Z}, - {"temperature", llama_sampler_type::TEMPERATURE} - }; + // process path from front to back, procedurally creating directories + while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) { + const std::wstring subpath = wpath.substr(0, pos_slash); + const wchar_t * test = subpath.c_str(); - // since samplers names are written multiple ways - // make it ready for both system names and input names - std::unordered_map sampler_alt_name_map { - {"top-k", llama_sampler_type::TOP_K}, - {"top-p", llama_sampler_type::TOP_P}, - {"nucleus", llama_sampler_type::TOP_P}, - {"typical-p", llama_sampler_type::TYPICAL_P}, - {"typical", llama_sampler_type::TYPICAL_P}, - {"min-p", llama_sampler_type::MIN_P}, - {"tfs-z", llama_sampler_type::TFS_Z}, - {"tfs", llama_sampler_type::TFS_Z}, - {"temp", llama_sampler_type::TEMPERATURE} - }; + const bool success = CreateDirectoryW(test, NULL); + if (!success) { + const DWORD error = GetLastError(); - std::vector sampler_types; - sampler_types.reserve(names.size()); - for (const auto & name : names) - { - auto sampler_item = sampler_canonical_name_map.find(name); - if (sampler_item != sampler_canonical_name_map.end()) - { - sampler_types.push_back(sampler_item->second); - } - else - { - if (allow_alt_names) - { - sampler_item = sampler_alt_name_map.find(name); - if (sampler_item != sampler_alt_name_map.end()) - { - sampler_types.push_back(sampler_item->second); + // if the path already exists, ensure that it's a directory + if (error == ERROR_ALREADY_EXISTS) { + const DWORD attributes = GetFileAttributesW(subpath.c_str()); + if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) { + return false; } + } else { + return false; } } + + pos_slash += 1; } - return sampler_types; -} -std::vector sampler_types_from_chars(const std::string & names_string) { - std::unordered_map sampler_name_map { - {'k', llama_sampler_type::TOP_K}, - {'p', llama_sampler_type::TOP_P}, - {'y', llama_sampler_type::TYPICAL_P}, - {'m', llama_sampler_type::MIN_P}, - {'f', llama_sampler_type::TFS_Z}, - {'t', llama_sampler_type::TEMPERATURE} - }; + return true; +#else + // if the path already exists, check whether it's a directory + struct stat info; + if (stat(path.c_str(), &info) == 0) { + return S_ISDIR(info.st_mode); + } - std::vector sampler_types; - sampler_types.reserve(names_string.size()); - for (const auto & c : names_string) { - const auto sampler_item = sampler_name_map.find(c); - if (sampler_item != sampler_name_map.end()) { - sampler_types.push_back(sampler_item->second); + size_t pos_slash = 1; // skip leading slashes for directory creation + + // process path from front to back, procedurally creating directories + while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) { + const std::string subpath = path.substr(0, pos_slash); + struct stat info; + + // if the path already exists, ensure that it's a directory + if (stat(subpath.c_str(), &info) == 0) { + if (!S_ISDIR(info.st_mode)) { + return false; + } + } else { + // create parent directories + const int ret = mkdir(subpath.c_str(), 0755); + if (ret != 0) { + return false; + } } + + pos_slash += 1; } - return sampler_types; + + return true; +#endif // _WIN32 } -std::string sampler_type_to_name_string(llama_sampler_type sampler_type) { - switch (sampler_type) { - case llama_sampler_type::TOP_K: return "top_k"; - case llama_sampler_type::TFS_Z: return "tfs_z"; - case llama_sampler_type::TYPICAL_P: return "typical_p"; - case llama_sampler_type::TOP_P: return "top_p"; - case llama_sampler_type::MIN_P: return "min_p"; - case llama_sampler_type::TEMPERATURE: return "temperature"; - default : return ""; +std::string fs_get_cache_directory() { + std::string cache_directory = ""; + if (getenv("LLAMA_CACHE")) { + cache_directory = std::getenv("LLAMA_CACHE"); + if (cache_directory.back() != DIRECTORY_SEPARATOR) { + cache_directory += DIRECTORY_SEPARATOR; + } + } else { +#ifdef __linux__ + if (std::getenv("XDG_CACHE_HOME")) { + cache_directory = std::getenv("XDG_CACHE_HOME"); + } else { + cache_directory = std::getenv("HOME") + std::string("/.cache/"); + } +#elif defined(__APPLE__) + cache_directory = std::getenv("HOME") + std::string("/Library/Caches/"); +#elif defined(_WIN32) + cache_directory = std::getenv("APPDATA"); +#endif // __linux__ + cache_directory += "llama.cpp"; + cache_directory += DIRECTORY_SEPARATOR; } + return cache_directory; } + // // Model utils // +std::tuple llama_init_from_gpt_params(gpt_params & params) { + auto mparams = llama_model_params_from_gpt_params(params); + + llama_model * model = nullptr; + + if (!params.hf_repo.empty() && !params.hf_file.empty()) { + model = llama_load_model_from_hf(params.hf_repo.c_str(), params.hf_file.c_str(), params.model.c_str(), mparams); + } else if (!params.model_url.empty()) { + model = llama_load_model_from_url(params.model_url.c_str(), params.model.c_str(), mparams); + } else { + model = llama_load_model_from_file(params.model.c_str(), mparams); + } + + if (model == NULL) { + fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str()); + return std::make_tuple(nullptr, nullptr); + } + + auto cparams = llama_context_params_from_gpt_params(params); + + llama_context * lctx = llama_new_context_with_model(model, cparams); + if (lctx == NULL) { + fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str()); + llama_free_model(model); + return std::make_tuple(nullptr, nullptr); + } + + if (!params.control_vectors.empty()) { + if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1; + if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_n_layer(model); + + const auto cvec = llama_control_vector_load(params.control_vectors); + if (cvec.n_embd == -1) { + llama_free(lctx); + llama_free_model(model); + return std::make_tuple(nullptr, nullptr); + } + + int err = llama_control_vector_apply(lctx, + cvec.data.data(), + cvec.data.size(), + cvec.n_embd, + params.control_vector_layer_start, + params.control_vector_layer_end); + if (err) { + llama_free(lctx); + llama_free_model(model); + return std::make_tuple(nullptr, nullptr); + } + } + + for (unsigned int i = 0; i < params.lora_adapter.size(); ++i) { + const std::string & lora_adapter = std::get<0>(params.lora_adapter[i]); + float lora_scale = std::get<1>(params.lora_adapter[i]); + int err = llama_model_apply_lora_from_file(model, + lora_adapter.c_str(), + lora_scale, + ((i > 0) || params.lora_base.empty()) + ? NULL + : params.lora_base.c_str(), + params.n_threads); + if (err != 0) { + fprintf(stderr, "%s: error: failed to apply lora adapter\n", __func__); + llama_free(lctx); + llama_free_model(model); + return std::make_tuple(nullptr, nullptr); + } + } + + if (params.ignore_eos) { + params.sparams.logit_bias[llama_token_eos(model)] = -INFINITY; + } + + if (params.warmup) { + LOG("warming up the model with an empty run\n"); + + std::vector tmp = { llama_token_bos(model), llama_token_eos(model), }; + llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0)); + llama_kv_cache_clear(lctx); + llama_synchronize(lctx); + llama_reset_timings(lctx); + } + + return std::make_tuple(model, lctx); +} + struct llama_model_params llama_model_params_from_gpt_params(const gpt_params & params) { auto mparams = llama_model_default_params(); @@ -1923,27 +2054,6 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param return cparams; } -void llama_batch_clear(struct llama_batch & batch) { - batch.n_tokens = 0; -} - -void llama_batch_add( - struct llama_batch & batch, - llama_token id, - llama_pos pos, - const std::vector & seq_ids, - bool logits) { - batch.token [batch.n_tokens] = id; - batch.pos [batch.n_tokens] = pos; - batch.n_seq_id[batch.n_tokens] = seq_ids.size(); - for (size_t i = 0; i < seq_ids.size(); ++i) { - batch.seq_id[batch.n_tokens][i] = seq_ids[i]; - } - batch.logits [batch.n_tokens] = logits; - - batch.n_tokens++; -} - #ifdef LLAMA_USE_CURL static bool starts_with(const std::string & str, const std::string & prefix) { @@ -2274,90 +2384,29 @@ struct llama_model * llama_load_model_from_hf( #endif // LLAMA_USE_CURL -std::tuple llama_init_from_gpt_params(gpt_params & params) { - auto mparams = llama_model_params_from_gpt_params(params); +// +// Batch utils +// - llama_model * model = nullptr; +void llama_batch_clear(struct llama_batch & batch) { + batch.n_tokens = 0; +} - if (!params.hf_repo.empty() && !params.hf_file.empty()) { - model = llama_load_model_from_hf(params.hf_repo.c_str(), params.hf_file.c_str(), params.model.c_str(), mparams); - } else if (!params.model_url.empty()) { - model = llama_load_model_from_url(params.model_url.c_str(), params.model.c_str(), mparams); - } else { - model = llama_load_model_from_file(params.model.c_str(), mparams); +void llama_batch_add( + struct llama_batch & batch, + llama_token id, + llama_pos pos, + const std::vector & seq_ids, + bool logits) { + batch.token [batch.n_tokens] = id; + batch.pos [batch.n_tokens] = pos; + batch.n_seq_id[batch.n_tokens] = seq_ids.size(); + for (size_t i = 0; i < seq_ids.size(); ++i) { + batch.seq_id[batch.n_tokens][i] = seq_ids[i]; } + batch.logits [batch.n_tokens] = logits; - if (model == NULL) { - fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str()); - return std::make_tuple(nullptr, nullptr); - } - - auto cparams = llama_context_params_from_gpt_params(params); - - llama_context * lctx = llama_new_context_with_model(model, cparams); - if (lctx == NULL) { - fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str()); - llama_free_model(model); - return std::make_tuple(nullptr, nullptr); - } - - if (!params.control_vectors.empty()) { - if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1; - if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_n_layer(model); - - const auto cvec = llama_control_vector_load(params.control_vectors); - if (cvec.n_embd == -1) { - llama_free(lctx); - llama_free_model(model); - return std::make_tuple(nullptr, nullptr); - } - - int err = llama_control_vector_apply(lctx, - cvec.data.data(), - cvec.data.size(), - cvec.n_embd, - params.control_vector_layer_start, - params.control_vector_layer_end); - if (err) { - llama_free(lctx); - llama_free_model(model); - return std::make_tuple(nullptr, nullptr); - } - } - - for (unsigned int i = 0; i < params.lora_adapter.size(); ++i) { - const std::string & lora_adapter = std::get<0>(params.lora_adapter[i]); - float lora_scale = std::get<1>(params.lora_adapter[i]); - int err = llama_model_apply_lora_from_file(model, - lora_adapter.c_str(), - lora_scale, - ((i > 0) || params.lora_base.empty()) - ? NULL - : params.lora_base.c_str(), - params.n_threads); - if (err != 0) { - fprintf(stderr, "%s: error: failed to apply lora adapter\n", __func__); - llama_free(lctx); - llama_free_model(model); - return std::make_tuple(nullptr, nullptr); - } - } - - if (params.ignore_eos) { - params.sparams.logit_bias[llama_token_eos(model)] = -INFINITY; - } - - if (params.warmup) { - LOG("warming up the model with an empty run\n"); - - std::vector tmp = { llama_token_bos(model), llama_token_eos(model), }; - llama_decode(lctx, llama_batch_get_one(tmp.data(), std::min(tmp.size(), (size_t) params.n_batch), 0, 0)); - llama_kv_cache_clear(lctx); - llama_synchronize(lctx); - llama_reset_timings(lctx); - } - - return std::make_tuple(model, lctx); + batch.n_tokens++; } // @@ -2445,346 +2494,11 @@ bool llama_should_add_bos_token(const llama_model * model) { return add_bos != -1 ? bool(add_bos) : (llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM); } -// -// YAML utils -// - -// returns true if successful, false otherwise -bool create_directory_with_parents(const std::string & path) { -#ifdef _WIN32 - std::wstring_convert> converter; - std::wstring wpath = converter.from_bytes(path); - - // if the path already exists, check whether it's a directory - const DWORD attributes = GetFileAttributesW(wpath.c_str()); - if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) { - return true; - } - - size_t pos_slash = 0; - - // process path from front to back, procedurally creating directories - while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) { - const std::wstring subpath = wpath.substr(0, pos_slash); - const wchar_t * test = subpath.c_str(); - - const bool success = CreateDirectoryW(test, NULL); - if (!success) { - const DWORD error = GetLastError(); - - // if the path already exists, ensure that it's a directory - if (error == ERROR_ALREADY_EXISTS) { - const DWORD attributes = GetFileAttributesW(subpath.c_str()); - if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) { - return false; - } - } else { - return false; - } - } - - pos_slash += 1; - } - - return true; -#else - // if the path already exists, check whether it's a directory - struct stat info; - if (stat(path.c_str(), &info) == 0) { - return S_ISDIR(info.st_mode); - } - - size_t pos_slash = 1; // skip leading slashes for directory creation - - // process path from front to back, procedurally creating directories - while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) { - const std::string subpath = path.substr(0, pos_slash); - struct stat info; - - // if the path already exists, ensure that it's a directory - if (stat(subpath.c_str(), &info) == 0) { - if (!S_ISDIR(info.st_mode)) { - return false; - } - } else { - // create parent directories - const int ret = mkdir(subpath.c_str(), 0755); - if (ret != 0) { - return false; - } - } - - pos_slash += 1; - } - - return true; -#endif // _WIN32 -} - -std::string get_cache_directory() { - std::string cache_directory = ""; - if (getenv("LLAMA_CACHE")) { - cache_directory = std::getenv("LLAMA_CACHE"); - if (cache_directory.back() != DIRECTORY_SEPARATOR) { - cache_directory += DIRECTORY_SEPARATOR; - } - } else { -#ifdef __linux__ - if (std::getenv("XDG_CACHE_HOME")) { - cache_directory = std::getenv("XDG_CACHE_HOME"); - } else { - cache_directory = std::getenv("HOME") + std::string("/.cache/"); - } -#elif defined(__APPLE__) - cache_directory = std::getenv("HOME") + std::string("/Library/Caches/"); -#elif defined(_WIN32) - cache_directory = std::getenv("APPDATA"); -#endif // __linux__ - cache_directory += "llama.cpp"; - cache_directory += DIRECTORY_SEPARATOR; - } - return cache_directory; -} - -void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector & data) { - if (data.empty()) { - fprintf(stream, "%s:\n", prop_name); - return; - } - - fprintf(stream, "%s: [", prop_name); - for (size_t i = 0; i < data.size() - 1; ++i) { - fprintf(stream, "%e, ", data[i]); - } - fprintf(stream, "%e]\n", data.back()); -} - -void dump_vector_int_yaml(FILE * stream, const char * prop_name, const std::vector & data) { - if (data.empty()) { - fprintf(stream, "%s:\n", prop_name); - return; - } - - fprintf(stream, "%s: [", prop_name); - for (size_t i = 0; i < data.size() - 1; ++i) { - fprintf(stream, "%d, ", data[i]); - } - fprintf(stream, "%d]\n", data.back()); -} - -void dump_string_yaml_multiline(FILE * stream, const char * prop_name, const char * data) { - std::string data_str(data == NULL ? "" : data); - - if (data_str.empty()) { - fprintf(stream, "%s:\n", prop_name); - return; - } - - size_t pos_start = 0; - size_t pos_found = 0; - - if (std::isspace(data_str[0]) || std::isspace(data_str.back())) { - data_str = std::regex_replace(data_str, std::regex("\n"), "\\n"); - data_str = std::regex_replace(data_str, std::regex("\""), "\\\""); - data_str = std::regex_replace(data_str, std::regex(R"(\\[^n"])"), R"(\$&)"); - data_str = "\"" + data_str + "\""; - fprintf(stream, "%s: %s\n", prop_name, data_str.c_str()); - return; - } - - if (data_str.find('\n') == std::string::npos) { - fprintf(stream, "%s: %s\n", prop_name, data_str.c_str()); - return; - } - - fprintf(stream, "%s: |\n", prop_name); - while ((pos_found = data_str.find('\n', pos_start)) != std::string::npos) { - fprintf(stream, " %s\n", data_str.substr(pos_start, pos_found-pos_start).c_str()); - pos_start = pos_found + 1; - } -} - -std::string get_sortable_timestamp() { - using clock = std::chrono::system_clock; - - const clock::time_point current_time = clock::now(); - const time_t as_time_t = clock::to_time_t(current_time); - char timestamp_no_ns[100]; - std::strftime(timestamp_no_ns, 100, "%Y_%m_%d-%H_%M_%S", std::localtime(&as_time_t)); - - const int64_t ns = std::chrono::duration_cast( - current_time.time_since_epoch() % 1000000000).count(); - char timestamp_ns[11]; - snprintf(timestamp_ns, 11, "%09" PRId64, ns); - - return std::string(timestamp_no_ns) + "." + std::string(timestamp_ns); -} - -void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const llama_context * lctx, - const std::string & timestamp, const std::vector & prompt_tokens, const char * model_desc) { - const llama_sampling_params & sparams = params.sparams; - - fprintf(stream, "build_commit: %s\n", LLAMA_COMMIT); - fprintf(stream, "build_number: %d\n", LLAMA_BUILD_NUMBER); - fprintf(stream, "cpu_has_arm_fma: %s\n", ggml_cpu_has_arm_fma() ? "true" : "false"); - fprintf(stream, "cpu_has_avx: %s\n", ggml_cpu_has_avx() ? "true" : "false"); - fprintf(stream, "cpu_has_avx_vnni: %s\n", ggml_cpu_has_avx_vnni() ? "true" : "false"); - fprintf(stream, "cpu_has_avx2: %s\n", ggml_cpu_has_avx2() ? "true" : "false"); - fprintf(stream, "cpu_has_avx512: %s\n", ggml_cpu_has_avx512() ? "true" : "false"); - fprintf(stream, "cpu_has_avx512_vbmi: %s\n", ggml_cpu_has_avx512_vbmi() ? "true" : "false"); - fprintf(stream, "cpu_has_avx512_vnni: %s\n", ggml_cpu_has_avx512_vnni() ? "true" : "false"); - fprintf(stream, "cpu_has_cuda: %s\n", ggml_cpu_has_cuda() ? "true" : "false"); - fprintf(stream, "cpu_has_vulkan: %s\n", ggml_cpu_has_vulkan() ? "true" : "false"); - fprintf(stream, "cpu_has_clblast: %s\n", ggml_cpu_has_clblast() ? "true" : "false"); - fprintf(stream, "cpu_has_kompute: %s\n", ggml_cpu_has_kompute() ? "true" : "false"); - fprintf(stream, "cpu_has_fma: %s\n", ggml_cpu_has_fma() ? "true" : "false"); - fprintf(stream, "cpu_has_gpublas: %s\n", ggml_cpu_has_gpublas() ? "true" : "false"); - fprintf(stream, "cpu_has_neon: %s\n", ggml_cpu_has_neon() ? "true" : "false"); - fprintf(stream, "cpu_has_f16c: %s\n", ggml_cpu_has_f16c() ? "true" : "false"); - fprintf(stream, "cpu_has_fp16_va: %s\n", ggml_cpu_has_fp16_va() ? "true" : "false"); - fprintf(stream, "cpu_has_wasm_simd: %s\n", ggml_cpu_has_wasm_simd() ? "true" : "false"); - fprintf(stream, "cpu_has_blas: %s\n", ggml_cpu_has_blas() ? "true" : "false"); - fprintf(stream, "cpu_has_sse3: %s\n", ggml_cpu_has_sse3() ? "true" : "false"); - fprintf(stream, "cpu_has_vsx: %s\n", ggml_cpu_has_vsx() ? "true" : "false"); - fprintf(stream, "cpu_has_matmul_int8: %s\n", ggml_cpu_has_matmul_int8() ? "true" : "false"); - -#ifdef NDEBUG - fprintf(stream, "debug: false\n"); -#else - fprintf(stream, "debug: true\n"); -#endif // NDEBUG - - fprintf(stream, "model_desc: %s\n", model_desc); - fprintf(stream, "n_vocab: %d # output size of the final layer, 32001 for some models\n", llama_n_vocab(llama_get_model(lctx))); - -#ifdef __OPTIMIZE__ - fprintf(stream, "optimize: true\n"); -#else - fprintf(stream, "optimize: false\n"); -#endif // __OPTIMIZE__ - - fprintf(stream, "time: %s\n", timestamp.c_str()); - - fprintf(stream, "\n"); - fprintf(stream, "###############\n"); - fprintf(stream, "# User Inputs #\n"); - fprintf(stream, "###############\n"); - fprintf(stream, "\n"); - - fprintf(stream, "alias: %s # default: unknown\n", params.model_alias.c_str()); - fprintf(stream, "batch_size: %d # default: 512\n", params.n_batch); - dump_string_yaml_multiline(stream, "cfg_negative_prompt", sparams.cfg_negative_prompt.c_str()); - fprintf(stream, "cfg_scale: %f # default: 1.0\n", sparams.cfg_scale); - fprintf(stream, "chunks: %d # default: -1 (unlimited)\n", params.n_chunks); - fprintf(stream, "color: %s # default: false\n", params.use_color ? "true" : "false"); - fprintf(stream, "ctx_size: %d # default: 512\n", params.n_ctx); - fprintf(stream, "escape: %s # default: false\n", params.escape ? "true" : "false"); - fprintf(stream, "file: # never logged, see prompt instead. Can still be specified for input.\n"); - fprintf(stream, "frequency_penalty: %f # default: 0.0 \n", sparams.penalty_freq); - dump_string_yaml_multiline(stream, "grammar", sparams.grammar.c_str()); - fprintf(stream, "grammar-file: # never logged, see grammar instead. Can still be specified for input.\n"); - fprintf(stream, "hellaswag: %s # default: false\n", params.hellaswag ? "true" : "false"); - fprintf(stream, "hellaswag_tasks: %zu # default: 400\n", params.hellaswag_tasks); - - const auto logit_bias_eos = sparams.logit_bias.find(llama_token_eos(llama_get_model(lctx))); - const bool ignore_eos = logit_bias_eos != sparams.logit_bias.end() && logit_bias_eos->second == -INFINITY; - fprintf(stream, "ignore_eos: %s # default: false\n", ignore_eos ? "true" : "false"); - - dump_string_yaml_multiline(stream, "in_prefix", params.input_prefix.c_str()); - fprintf(stream, "in_prefix_bos: %s # default: false\n", params.input_prefix_bos ? "true" : "false"); - dump_string_yaml_multiline(stream, "in_suffix", params.input_prefix.c_str()); - fprintf(stream, "instruct: %s # default: false\n", params.instruct ? "true" : "false"); - fprintf(stream, "interactive: %s # default: false\n", params.interactive ? "true" : "false"); - fprintf(stream, "interactive_specials: %s # default: false\n", params.interactive_specials ? "true" : "false"); - fprintf(stream, "interactive_first: %s # default: false\n", params.interactive_first ? "true" : "false"); - fprintf(stream, "keep: %d # default: 0\n", params.n_keep); - fprintf(stream, "logdir: %s # default: unset (no logging)\n", params.logdir.c_str()); - - fprintf(stream, "logit_bias:\n"); - for (std::pair lb : sparams.logit_bias) { - if (ignore_eos && lb.first == logit_bias_eos->first) { - continue; - } - fprintf(stream, " %d: %f", lb.first, lb.second); - } - - fprintf(stream, "lora:\n"); - for (std::tuple la : params.lora_adapter) { - if (std::get<1>(la) != 1.0f) { - continue; - } - fprintf(stream, " - %s\n", std::get<0>(la).c_str()); - } - fprintf(stream, "lora_scaled:\n"); - for (std::tuple la : params.lora_adapter) { - if (std::get<1>(la) == 1.0f) { - continue; - } - fprintf(stream, " - %s: %f\n", std::get<0>(la).c_str(), std::get<1>(la)); - } - fprintf(stream, "lora_base: %s\n", params.lora_base.c_str()); - fprintf(stream, "main_gpu: %d # default: 0\n", params.main_gpu); - fprintf(stream, "min_keep: %d # default: 0 (disabled)\n", sparams.min_keep); - fprintf(stream, "mirostat: %d # default: 0 (disabled)\n", sparams.mirostat); - fprintf(stream, "mirostat_ent: %f # default: 5.0\n", sparams.mirostat_tau); - fprintf(stream, "mirostat_lr: %f # default: 0.1\n", sparams.mirostat_eta); - fprintf(stream, "mlock: %s # default: false\n", params.use_mlock ? "true" : "false"); - fprintf(stream, "model: %s # default: %s\n", params.model.c_str(), DEFAULT_MODEL_PATH); - fprintf(stream, "model_draft: %s # default:\n", params.model_draft.c_str()); - fprintf(stream, "multiline_input: %s # default: false\n", params.multiline_input ? "true" : "false"); - fprintf(stream, "n_gpu_layers: %d # default: -1\n", params.n_gpu_layers); - fprintf(stream, "n_predict: %d # default: -1 (unlimited)\n", params.n_predict); - fprintf(stream, "n_probs: %d # only used by server binary, default: 0\n", sparams.n_probs); - fprintf(stream, "no_mmap: %s # default: false\n", !params.use_mmap ? "true" : "false"); - fprintf(stream, "penalize_nl: %s # default: false\n", sparams.penalize_nl ? "true" : "false"); - fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type); - fprintf(stream, "ppl_stride: %d # default: 0\n", params.ppl_stride); - fprintf(stream, "presence_penalty: %f # default: 0.0\n", sparams.penalty_present); - dump_string_yaml_multiline(stream, "prompt", params.prompt.c_str()); - fprintf(stream, "prompt_cache: %s\n", params.path_prompt_cache.c_str()); - fprintf(stream, "prompt_cache_all: %s # default: false\n", params.prompt_cache_all ? "true" : "false"); - fprintf(stream, "prompt_cache_ro: %s # default: false\n", params.prompt_cache_ro ? "true" : "false"); - dump_vector_int_yaml(stream, "prompt_tokens", prompt_tokens); - fprintf(stream, "random_prompt: %s # default: false\n", params.random_prompt ? "true" : "false"); - fprintf(stream, "repeat_penalty: %f # default: 1.1\n", sparams.penalty_repeat); - - fprintf(stream, "reverse_prompt:\n"); - for (std::string ap : params.antiprompt) { - size_t pos = 0; - while ((pos = ap.find('\n', pos)) != std::string::npos) { - ap.replace(pos, 1, "\\n"); - pos += 1; - } - - fprintf(stream, " - %s\n", ap.c_str()); - } - - fprintf(stream, "rope_freq_base: %f # default: 10000.0\n", params.rope_freq_base); - fprintf(stream, "rope_freq_scale: %f # default: 1.0\n", params.rope_freq_scale); - fprintf(stream, "seed: %u # default: -1 (random seed)\n", params.seed); - fprintf(stream, "simple_io: %s # default: false\n", params.simple_io ? "true" : "false"); - fprintf(stream, "cont_batching: %s # default: false\n", params.cont_batching ? "true" : "false"); - fprintf(stream, "flash_attn: %s # default: false\n", params.flash_attn ? "true" : "false"); - fprintf(stream, "temp: %f # default: 0.8\n", sparams.temp); - - const std::vector tensor_split_vector(params.tensor_split, params.tensor_split + llama_max_devices()); - dump_vector_float_yaml(stream, "tensor_split", tensor_split_vector); - - fprintf(stream, "tfs: %f # default: 1.0\n", sparams.tfs_z); - fprintf(stream, "threads: %d # default: %u\n", params.n_threads, std::thread::hardware_concurrency()); - fprintf(stream, "top_k: %d # default: 40\n", sparams.top_k); - fprintf(stream, "top_p: %f # default: 0.95\n", sparams.top_p); - fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p); - fprintf(stream, "typical_p: %f # default: 1.0\n", sparams.typical_p); - fprintf(stream, "verbose_prompt: %s # default: false\n", params.verbose_prompt ? "true" : "false"); - fprintf(stream, "display_prompt: %s # default: true\n", params.display_prompt ? "true" : "false"); -} - // // KV cache utils // -void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size) { +void llama_kv_cache_dump_view(const llama_kv_cache_view & view, int row_size) { static const char slot_chars[] = ".123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz+"; printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d", @@ -2807,7 +2521,7 @@ void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size) { printf("\n=== Done dumping\n"); } -void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) { +void llama_kv_cache_dump_view_seqs(const llama_kv_cache_view & view, int row_size) { static const char slot_chars[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; printf("=== Dumping KV cache. total cells %d, max sequences per cell %d, populated cells %d, total tokens in cache %d, largest empty slot=%d @ %d\n", @@ -2855,6 +2569,10 @@ void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size) { printf("\n=== Done dumping\n"); } +// +// Embedding utils +// + void llama_embd_normalize(const float * inp, float * out, int n) { double sum = 0.0; for (int i = 0; i < n; i++) { @@ -3039,3 +2757,225 @@ llama_control_vector_data llama_control_vector_load(const std::vector & data) { + if (data.empty()) { + fprintf(stream, "%s:\n", prop_name); + return; + } + + fprintf(stream, "%s: [", prop_name); + for (size_t i = 0; i < data.size() - 1; ++i) { + fprintf(stream, "%e, ", data[i]); + } + fprintf(stream, "%e]\n", data.back()); +} + +void yaml_dump_vector_int(FILE * stream, const char * prop_name, const std::vector & data) { + if (data.empty()) { + fprintf(stream, "%s:\n", prop_name); + return; + } + + fprintf(stream, "%s: [", prop_name); + for (size_t i = 0; i < data.size() - 1; ++i) { + fprintf(stream, "%d, ", data[i]); + } + fprintf(stream, "%d]\n", data.back()); +} + +void yaml_dump_string_multiline(FILE * stream, const char * prop_name, const char * data) { + std::string data_str(data == NULL ? "" : data); + + if (data_str.empty()) { + fprintf(stream, "%s:\n", prop_name); + return; + } + + size_t pos_start = 0; + size_t pos_found = 0; + + if (std::isspace(data_str[0]) || std::isspace(data_str.back())) { + data_str = std::regex_replace(data_str, std::regex("\n"), "\\n"); + data_str = std::regex_replace(data_str, std::regex("\""), "\\\""); + data_str = std::regex_replace(data_str, std::regex(R"(\\[^n"])"), R"(\$&)"); + data_str = "\"" + data_str + "\""; + fprintf(stream, "%s: %s\n", prop_name, data_str.c_str()); + return; + } + + if (data_str.find('\n') == std::string::npos) { + fprintf(stream, "%s: %s\n", prop_name, data_str.c_str()); + return; + } + + fprintf(stream, "%s: |\n", prop_name); + while ((pos_found = data_str.find('\n', pos_start)) != std::string::npos) { + fprintf(stream, " %s\n", data_str.substr(pos_start, pos_found-pos_start).c_str()); + pos_start = pos_found + 1; + } +} + +void yaml_dump_non_result_info(FILE * stream, const gpt_params & params, const llama_context * lctx, + const std::string & timestamp, const std::vector & prompt_tokens, const char * model_desc) { + const llama_sampling_params & sparams = params.sparams; + + fprintf(stream, "build_commit: %s\n", LLAMA_COMMIT); + fprintf(stream, "build_number: %d\n", LLAMA_BUILD_NUMBER); + fprintf(stream, "cpu_has_arm_fma: %s\n", ggml_cpu_has_arm_fma() ? "true" : "false"); + fprintf(stream, "cpu_has_avx: %s\n", ggml_cpu_has_avx() ? "true" : "false"); + fprintf(stream, "cpu_has_avx_vnni: %s\n", ggml_cpu_has_avx_vnni() ? "true" : "false"); + fprintf(stream, "cpu_has_avx2: %s\n", ggml_cpu_has_avx2() ? "true" : "false"); + fprintf(stream, "cpu_has_avx512: %s\n", ggml_cpu_has_avx512() ? "true" : "false"); + fprintf(stream, "cpu_has_avx512_vbmi: %s\n", ggml_cpu_has_avx512_vbmi() ? "true" : "false"); + fprintf(stream, "cpu_has_avx512_vnni: %s\n", ggml_cpu_has_avx512_vnni() ? "true" : "false"); + fprintf(stream, "cpu_has_cuda: %s\n", ggml_cpu_has_cuda() ? "true" : "false"); + fprintf(stream, "cpu_has_vulkan: %s\n", ggml_cpu_has_vulkan() ? "true" : "false"); + fprintf(stream, "cpu_has_clblast: %s\n", ggml_cpu_has_clblast() ? "true" : "false"); + fprintf(stream, "cpu_has_kompute: %s\n", ggml_cpu_has_kompute() ? "true" : "false"); + fprintf(stream, "cpu_has_fma: %s\n", ggml_cpu_has_fma() ? "true" : "false"); + fprintf(stream, "cpu_has_gpublas: %s\n", ggml_cpu_has_gpublas() ? "true" : "false"); + fprintf(stream, "cpu_has_neon: %s\n", ggml_cpu_has_neon() ? "true" : "false"); + fprintf(stream, "cpu_has_f16c: %s\n", ggml_cpu_has_f16c() ? "true" : "false"); + fprintf(stream, "cpu_has_fp16_va: %s\n", ggml_cpu_has_fp16_va() ? "true" : "false"); + fprintf(stream, "cpu_has_wasm_simd: %s\n", ggml_cpu_has_wasm_simd() ? "true" : "false"); + fprintf(stream, "cpu_has_blas: %s\n", ggml_cpu_has_blas() ? "true" : "false"); + fprintf(stream, "cpu_has_sse3: %s\n", ggml_cpu_has_sse3() ? "true" : "false"); + fprintf(stream, "cpu_has_vsx: %s\n", ggml_cpu_has_vsx() ? "true" : "false"); + fprintf(stream, "cpu_has_matmul_int8: %s\n", ggml_cpu_has_matmul_int8() ? "true" : "false"); + +#ifdef NDEBUG + fprintf(stream, "debug: false\n"); +#else + fprintf(stream, "debug: true\n"); +#endif // NDEBUG + + fprintf(stream, "model_desc: %s\n", model_desc); + fprintf(stream, "n_vocab: %d # output size of the final layer, 32001 for some models\n", llama_n_vocab(llama_get_model(lctx))); + +#ifdef __OPTIMIZE__ + fprintf(stream, "optimize: true\n"); +#else + fprintf(stream, "optimize: false\n"); +#endif // __OPTIMIZE__ + + fprintf(stream, "time: %s\n", timestamp.c_str()); + + fprintf(stream, "\n"); + fprintf(stream, "###############\n"); + fprintf(stream, "# User Inputs #\n"); + fprintf(stream, "###############\n"); + fprintf(stream, "\n"); + + fprintf(stream, "alias: %s # default: unknown\n", params.model_alias.c_str()); + fprintf(stream, "batch_size: %d # default: 512\n", params.n_batch); + yaml_dump_string_multiline(stream, "cfg_negative_prompt", sparams.cfg_negative_prompt.c_str()); + fprintf(stream, "cfg_scale: %f # default: 1.0\n", sparams.cfg_scale); + fprintf(stream, "chunks: %d # default: -1 (unlimited)\n", params.n_chunks); + fprintf(stream, "color: %s # default: false\n", params.use_color ? "true" : "false"); + fprintf(stream, "ctx_size: %d # default: 512\n", params.n_ctx); + fprintf(stream, "escape: %s # default: false\n", params.escape ? "true" : "false"); + fprintf(stream, "file: # never logged, see prompt instead. Can still be specified for input.\n"); + fprintf(stream, "frequency_penalty: %f # default: 0.0 \n", sparams.penalty_freq); + yaml_dump_string_multiline(stream, "grammar", sparams.grammar.c_str()); + fprintf(stream, "grammar-file: # never logged, see grammar instead. Can still be specified for input.\n"); + fprintf(stream, "hellaswag: %s # default: false\n", params.hellaswag ? "true" : "false"); + fprintf(stream, "hellaswag_tasks: %zu # default: 400\n", params.hellaswag_tasks); + + const auto logit_bias_eos = sparams.logit_bias.find(llama_token_eos(llama_get_model(lctx))); + const bool ignore_eos = logit_bias_eos != sparams.logit_bias.end() && logit_bias_eos->second == -INFINITY; + fprintf(stream, "ignore_eos: %s # default: false\n", ignore_eos ? "true" : "false"); + + yaml_dump_string_multiline(stream, "in_prefix", params.input_prefix.c_str()); + fprintf(stream, "in_prefix_bos: %s # default: false\n", params.input_prefix_bos ? "true" : "false"); + yaml_dump_string_multiline(stream, "in_suffix", params.input_prefix.c_str()); + fprintf(stream, "instruct: %s # default: false\n", params.instruct ? "true" : "false"); + fprintf(stream, "interactive: %s # default: false\n", params.interactive ? "true" : "false"); + fprintf(stream, "interactive_specials: %s # default: false\n", params.interactive_specials ? "true" : "false"); + fprintf(stream, "interactive_first: %s # default: false\n", params.interactive_first ? "true" : "false"); + fprintf(stream, "keep: %d # default: 0\n", params.n_keep); + fprintf(stream, "logdir: %s # default: unset (no logging)\n", params.logdir.c_str()); + + fprintf(stream, "logit_bias:\n"); + for (std::pair lb : sparams.logit_bias) { + if (ignore_eos && lb.first == logit_bias_eos->first) { + continue; + } + fprintf(stream, " %d: %f", lb.first, lb.second); + } + + fprintf(stream, "lora:\n"); + for (std::tuple la : params.lora_adapter) { + if (std::get<1>(la) != 1.0f) { + continue; + } + fprintf(stream, " - %s\n", std::get<0>(la).c_str()); + } + fprintf(stream, "lora_scaled:\n"); + for (std::tuple la : params.lora_adapter) { + if (std::get<1>(la) == 1.0f) { + continue; + } + fprintf(stream, " - %s: %f\n", std::get<0>(la).c_str(), std::get<1>(la)); + } + fprintf(stream, "lora_base: %s\n", params.lora_base.c_str()); + fprintf(stream, "main_gpu: %d # default: 0\n", params.main_gpu); + fprintf(stream, "min_keep: %d # default: 0 (disabled)\n", sparams.min_keep); + fprintf(stream, "mirostat: %d # default: 0 (disabled)\n", sparams.mirostat); + fprintf(stream, "mirostat_ent: %f # default: 5.0\n", sparams.mirostat_tau); + fprintf(stream, "mirostat_lr: %f # default: 0.1\n", sparams.mirostat_eta); + fprintf(stream, "mlock: %s # default: false\n", params.use_mlock ? "true" : "false"); + fprintf(stream, "model: %s # default: %s\n", params.model.c_str(), DEFAULT_MODEL_PATH); + fprintf(stream, "model_draft: %s # default:\n", params.model_draft.c_str()); + fprintf(stream, "multiline_input: %s # default: false\n", params.multiline_input ? "true" : "false"); + fprintf(stream, "n_gpu_layers: %d # default: -1\n", params.n_gpu_layers); + fprintf(stream, "n_predict: %d # default: -1 (unlimited)\n", params.n_predict); + fprintf(stream, "n_probs: %d # only used by server binary, default: 0\n", sparams.n_probs); + fprintf(stream, "no_mmap: %s # default: false\n", !params.use_mmap ? "true" : "false"); + fprintf(stream, "penalize_nl: %s # default: false\n", sparams.penalize_nl ? "true" : "false"); + fprintf(stream, "ppl_output_type: %d # default: 0\n", params.ppl_output_type); + fprintf(stream, "ppl_stride: %d # default: 0\n", params.ppl_stride); + fprintf(stream, "presence_penalty: %f # default: 0.0\n", sparams.penalty_present); + yaml_dump_string_multiline(stream, "prompt", params.prompt.c_str()); + fprintf(stream, "prompt_cache: %s\n", params.path_prompt_cache.c_str()); + fprintf(stream, "prompt_cache_all: %s # default: false\n", params.prompt_cache_all ? "true" : "false"); + fprintf(stream, "prompt_cache_ro: %s # default: false\n", params.prompt_cache_ro ? "true" : "false"); + yaml_dump_vector_int(stream, "prompt_tokens", prompt_tokens); + fprintf(stream, "random_prompt: %s # default: false\n", params.random_prompt ? "true" : "false"); + fprintf(stream, "repeat_penalty: %f # default: 1.1\n", sparams.penalty_repeat); + + fprintf(stream, "reverse_prompt:\n"); + for (std::string ap : params.antiprompt) { + size_t pos = 0; + while ((pos = ap.find('\n', pos)) != std::string::npos) { + ap.replace(pos, 1, "\\n"); + pos += 1; + } + + fprintf(stream, " - %s\n", ap.c_str()); + } + + fprintf(stream, "rope_freq_base: %f # default: 10000.0\n", params.rope_freq_base); + fprintf(stream, "rope_freq_scale: %f # default: 1.0\n", params.rope_freq_scale); + fprintf(stream, "seed: %u # default: -1 (random seed)\n", params.seed); + fprintf(stream, "simple_io: %s # default: false\n", params.simple_io ? "true" : "false"); + fprintf(stream, "cont_batching: %s # default: false\n", params.cont_batching ? "true" : "false"); + fprintf(stream, "flash_attn: %s # default: false\n", params.flash_attn ? "true" : "false"); + fprintf(stream, "temp: %f # default: 0.8\n", sparams.temp); + + const std::vector tensor_split_vector(params.tensor_split, params.tensor_split + llama_max_devices()); + yaml_dump_vector_float(stream, "tensor_split", tensor_split_vector); + + fprintf(stream, "tfs: %f # default: 1.0\n", sparams.tfs_z); + fprintf(stream, "threads: %d # default: %u\n", params.n_threads, std::thread::hardware_concurrency()); + fprintf(stream, "top_k: %d # default: 40\n", sparams.top_k); + fprintf(stream, "top_p: %f # default: 0.95\n", sparams.top_p); + fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p); + fprintf(stream, "typical_p: %f # default: 1.0\n", sparams.typical_p); + fprintf(stream, "verbose_prompt: %s # default: false\n", params.verbose_prompt ? "true" : "false"); + fprintf(stream, "display_prompt: %s # default: true\n", params.display_prompt ? "true" : "false"); +} diff --git a/common/common.h b/common/common.h index a8e5e50e6..f68f3c297 100644 --- a/common/common.h +++ b/common/common.h @@ -27,7 +27,7 @@ #define die_fmt(fmt, ...) do { fprintf(stderr, "error: " fmt "\n", __VA_ARGS__); exit(1); } while (0) #define print_build_info() do { \ - fprintf(stderr, "%s: build = %d (%s)\n", __func__, LLAMA_BUILD_NUMBER, LLAMA_COMMIT); \ + fprintf(stderr, "%s: build = %d (%s)\n", __func__, LLAMA_BUILD_NUMBER, LLAMA_COMMIT); \ fprintf(stderr, "%s: built with %s for %s\n", __func__, LLAMA_COMPILER, LLAMA_BUILD_TARGET); \ } while(0) @@ -35,14 +35,18 @@ // build info extern int LLAMA_BUILD_NUMBER; -extern char const *LLAMA_COMMIT; -extern char const *LLAMA_COMPILER; -extern char const *LLAMA_BUILD_TARGET; +extern char const * LLAMA_COMMIT; +extern char const * LLAMA_COMPILER; +extern char const * LLAMA_BUILD_TARGET; struct llama_control_vector_load_info; -int get_math_cpu_count(); -int32_t get_num_physical_cores(); +// +// CPU utils +// + +int32_t cpu_get_num_physical_cores(); +int32_t cpu_get_num_math(); // // CLI argument parsing @@ -51,7 +55,7 @@ int32_t get_num_physical_cores(); struct gpt_params { uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed - int32_t n_threads = get_math_cpu_count(); + int32_t n_threads = cpu_get_num_math(); int32_t n_threads_draft = -1; int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads) int32_t n_threads_batch_draft = -1; @@ -179,33 +183,34 @@ struct gpt_params { void gpt_params_handle_model_default(gpt_params & params); -bool parse_kv_override(const char * data, std::vector & overrides); +bool gpt_params_parse_ex (int argc, char ** argv, gpt_params & params); +bool gpt_params_parse (int argc, char ** argv, gpt_params & params); +bool gpt_params_find_arg (int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param); +void gpt_params_print_usage(int argc, char ** argv, const gpt_params & params); -bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params); - -bool gpt_params_parse(int argc, char ** argv, gpt_params & params); - -void gpt_print_usage(int argc, char ** argv, const gpt_params & params); - -bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_params & params, int & i, bool & invalid_param); - -std::string get_system_info(const gpt_params & params); - -std::string gpt_random_prompt(std::mt19937 & rng); - -void process_escapes(std::string& input); - -bool validate_file_name(const std::string & filename); +std::string gpt_params_get_system_info(const gpt_params & params); // // String utils // -std::vector sampler_types_from_names(const std::vector & names, bool allow_alt_names); -std::vector sampler_types_from_chars(const std::string & names_string); std::vector string_split(std::string input, char separator); + std::string string_strip(const std::string & str); -std::string sampler_type_to_name_string(llama_sampler_type sampler_type); +std::string string_get_sortable_timestamp(); +std::string string_random_prompt(std::mt19937 & rng); + +bool string_parse_kv_override(const char * data, std::vector & overrides); +void string_process_escapes(std::string & input); + +// +// Filesystem utils +// + +bool fs_validate_filename(const std::string & filename); +bool fs_create_directory_with_parents(const std::string & path); + +std::string fs_get_cache_directory(); // // Model utils @@ -276,30 +281,15 @@ std::string llama_detokenize_bpe( // defaults to true when model type is SPM, otherwise false. bool llama_should_add_bos_token(const llama_model * model); -// -// YAML utils -// - -bool create_directory_with_parents(const std::string & path); -std::string get_cache_directory(); -void dump_vector_float_yaml(FILE * stream, const char * prop_name, const std::vector & data); -void dump_vector_int_yaml(FILE * stream, const char * prop_name, const std::vector & data); -void dump_string_yaml_multiline(FILE * stream, const char * prop_name, const char * data); -std::string get_sortable_timestamp(); - -void dump_non_result_info_yaml( - FILE * stream, const gpt_params & params, const llama_context * lctx, - const std::string & timestamp, const std::vector & prompt_tokens, const char * model_desc); - // // KV cache utils // // Dump the KV cache view with the number of sequences per cell. -void dump_kv_cache_view(const llama_kv_cache_view & view, int row_size = 80); +void llama_kv_cache_dump_view(const llama_kv_cache_view & view, int row_size = 80); // Dump the KV cache view showing individual sequences in each cell (long output). -void dump_kv_cache_view_seqs(const llama_kv_cache_view & view, int row_size = 40); +void llama_kv_cache_dump_view_seqs(const llama_kv_cache_view & view, int row_size = 40); // // Embedding utils @@ -333,6 +323,20 @@ llama_control_vector_data llama_control_vector_load(const std::vector & data); +void yaml_dump_vector_int (FILE * stream, const char * prop_name, const std::vector & data); +void yaml_dump_string_multiline(FILE * stream, const char * prop_name, const char * data); + +void yaml_dump_non_result_info( + FILE * stream, const gpt_params & params, const llama_context * lctx, + const std::string & timestamp, const std::vector & prompt_tokens, const char * model_desc); + diff --git a/common/sampling.cpp b/common/sampling.cpp index 7fc2e2158..f1f803516 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -125,7 +125,7 @@ std::string llama_sampling_order_print(const llama_sampling_params & params) { std::string result = "CFG -> Penalties "; if (params.mirostat == 0) { for (auto sampler_type : params.samplers_sequence) { - const auto sampler_type_name = sampler_type_to_name_string(sampler_type); + const auto sampler_type_name = llama_sampling_type_to_str(sampler_type); if (!sampler_type_name.empty()) { result += "-> " + sampler_type_name + " "; } @@ -137,6 +137,87 @@ std::string llama_sampling_order_print(const llama_sampling_params & params) { return result; } +std::string llama_sampling_type_to_str(llama_sampler_type sampler_type) { + switch (sampler_type) { + case llama_sampler_type::TOP_K: return "top_k"; + case llama_sampler_type::TFS_Z: return "tfs_z"; + case llama_sampler_type::TYPICAL_P: return "typical_p"; + case llama_sampler_type::TOP_P: return "top_p"; + case llama_sampler_type::MIN_P: return "min_p"; + case llama_sampler_type::TEMPERATURE: return "temperature"; + default : return ""; + } +} + +std::vector llama_sampling_types_from_names(const std::vector & names, bool allow_alt_names) { + std::unordered_map sampler_canonical_name_map { + {"top_k", llama_sampler_type::TOP_K}, + {"top_p", llama_sampler_type::TOP_P}, + {"typical_p", llama_sampler_type::TYPICAL_P}, + {"min_p", llama_sampler_type::MIN_P}, + {"tfs_z", llama_sampler_type::TFS_Z}, + {"temperature", llama_sampler_type::TEMPERATURE} + }; + + // since samplers names are written multiple ways + // make it ready for both system names and input names + std::unordered_map sampler_alt_name_map { + {"top-k", llama_sampler_type::TOP_K}, + {"top-p", llama_sampler_type::TOP_P}, + {"nucleus", llama_sampler_type::TOP_P}, + {"typical-p", llama_sampler_type::TYPICAL_P}, + {"typical", llama_sampler_type::TYPICAL_P}, + {"min-p", llama_sampler_type::MIN_P}, + {"tfs-z", llama_sampler_type::TFS_Z}, + {"tfs", llama_sampler_type::TFS_Z}, + {"temp", llama_sampler_type::TEMPERATURE} + }; + + std::vector sampler_types; + sampler_types.reserve(names.size()); + for (const auto & name : names) + { + auto sampler_item = sampler_canonical_name_map.find(name); + if (sampler_item != sampler_canonical_name_map.end()) + { + sampler_types.push_back(sampler_item->second); + } + else + { + if (allow_alt_names) + { + sampler_item = sampler_alt_name_map.find(name); + if (sampler_item != sampler_alt_name_map.end()) + { + sampler_types.push_back(sampler_item->second); + } + } + } + } + return sampler_types; +} + +std::vector llama_sampling_types_from_chars(const std::string & names_string) { + std::unordered_map sampler_name_map { + {'k', llama_sampler_type::TOP_K}, + {'p', llama_sampler_type::TOP_P}, + {'y', llama_sampler_type::TYPICAL_P}, + {'m', llama_sampler_type::MIN_P}, + {'f', llama_sampler_type::TFS_Z}, + {'t', llama_sampler_type::TEMPERATURE} + }; + + std::vector sampler_types; + sampler_types.reserve(names_string.size()); + for (const auto & c : names_string) { + const auto sampler_item = sampler_name_map.find(c); + if (sampler_item != sampler_name_map.end()) { + sampler_types.push_back(sampler_item->second); + } + } + return sampler_types; +} + // no reasons to expose this function in header static void sampler_queue( struct llama_context * ctx_main, diff --git a/common/sampling.h b/common/sampling.h index 655732ad1..eeaa53b8b 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -116,6 +116,11 @@ std::string llama_sampling_print(const llama_sampling_params & params); // Print sampling order into a string std::string llama_sampling_order_print(const llama_sampling_params & params); +std::string llama_sampling_type_to_str(llama_sampler_type sampler_type); + +std::vector llama_sampling_types_from_names(const std::vector & names, bool allow_alt_names); +std::vector llama_sampling_types_from_chars(const std::string & names_string); + // this is a common sampling function used across the examples for convenience // it can serve as a starting point for implementing your own sampling function // Note: When using multiple sequences, it is the caller's responsibility to call diff --git a/common/train.cpp b/common/train.cpp index 0dbfd24df..2d41a1d29 100644 --- a/common/train.cpp +++ b/common/train.cpp @@ -1380,7 +1380,7 @@ bool consume_common_train_arg( void finish_processing_train_args(struct train_params_common * params) { if (params->escape) { - process_escapes(params->sample_start); + string_process_escapes(params->sample_start); } } diff --git a/examples/batched/batched.cpp b/examples/batched/batched.cpp index be30d20bf..591bc6e57 100644 --- a/examples/batched/batched.cpp +++ b/examples/batched/batched.cpp @@ -48,7 +48,7 @@ int main(int argc, char ** argv) { params.prompt = "Hello my name is"; } - process_escapes(params.prompt); + string_process_escapes(params.prompt); // init LLM diff --git a/examples/embedding/embedding.cpp b/examples/embedding/embedding.cpp index 0c921ed69..004399b5f 100644 --- a/examples/embedding/embedding.cpp +++ b/examples/embedding/embedding.cpp @@ -80,7 +80,7 @@ int main(int argc, char ** argv) { std::mt19937 rng(params.seed); if (params.random_prompt) { - params.prompt = gpt_random_prompt(rng); + params.prompt = string_random_prompt(rng); } llama_backend_init(); @@ -107,7 +107,7 @@ int main(int argc, char ** argv) { // print system information { fprintf(stderr, "\n"); - fprintf(stderr, "%s\n", get_system_info(params).c_str()); + fprintf(stderr, "%s\n", gpt_params_get_system_info(params).c_str()); } // split the prompt into lines diff --git a/examples/eval-callback/eval-callback.cpp b/examples/eval-callback/eval-callback.cpp index e670d3769..51d67d6d9 100644 --- a/examples/eval-callback/eval-callback.cpp +++ b/examples/eval-callback/eval-callback.cpp @@ -152,7 +152,7 @@ int main(int argc, char ** argv) { std::mt19937 rng(params.seed); if (params.random_prompt) { - params.prompt = gpt_random_prompt(rng); + params.prompt = string_random_prompt(rng); } llama_backend_init(); @@ -176,7 +176,7 @@ int main(int argc, char ** argv) { // print system information { fprintf(stderr, "\n"); - fprintf(stderr, "%s\n", get_system_info(params).c_str()); + fprintf(stderr, "%s\n", gpt_params_get_system_info(params).c_str()); } bool OK = run(ctx, params); diff --git a/examples/imatrix/imatrix.cpp b/examples/imatrix/imatrix.cpp index 82b19fc4f..25a2351cc 100644 --- a/examples/imatrix/imatrix.cpp +++ b/examples/imatrix/imatrix.cpp @@ -598,7 +598,7 @@ int main(int argc, char ** argv) { std::mt19937 rng(params.seed); if (params.random_prompt) { - params.prompt = gpt_random_prompt(rng); + params.prompt = string_random_prompt(rng); } sparams.dataset = params.prompt_file; @@ -667,7 +667,7 @@ int main(int argc, char ** argv) { // print system information { fprintf(stderr, "\n"); - fprintf(stderr, "%s\n", get_system_info(params).c_str()); + fprintf(stderr, "%s\n", gpt_params_get_system_info(params).c_str()); } bool OK = compute_imatrix(ctx, params, compute_ppl, from_chunk); diff --git a/examples/infill/infill.cpp b/examples/infill/infill.cpp index afac145f6..539f78184 100644 --- a/examples/infill/infill.cpp +++ b/examples/infill/infill.cpp @@ -50,9 +50,9 @@ static void write_logfile( return; } - const std::string timestamp = get_sortable_timestamp(); + const std::string timestamp = string_get_sortable_timestamp(); - const bool success = create_directory_with_parents(params.logdir); + const bool success = fs_create_directory_with_parents(params.logdir); if (!success) { fprintf(stderr, "%s: warning: failed to create logdir %s, cannot write logfile\n", __func__, params.logdir.c_str()); @@ -70,7 +70,7 @@ static void write_logfile( fprintf(logfile, "binary: infill\n"); char model_desc[128]; llama_model_desc(model, model_desc, sizeof(model_desc)); - dump_non_result_info_yaml(logfile, params, ctx, timestamp, input_tokens, model_desc); + yaml_dump_non_result_info(logfile, params, ctx, timestamp, input_tokens, model_desc); fprintf(logfile, "\n"); fprintf(logfile, "######################\n"); @@ -78,8 +78,8 @@ static void write_logfile( fprintf(logfile, "######################\n"); fprintf(logfile, "\n"); - dump_string_yaml_multiline(logfile, "output", output.c_str()); - dump_vector_int_yaml(logfile, "output_tokens", output_tokens); + yaml_dump_string_multiline(logfile, "output", output.c_str()); + yaml_dump_vector_int(logfile, "output_tokens", output_tokens); llama_dump_timing_info_yaml(logfile, ctx); fclose(logfile); @@ -236,7 +236,7 @@ int main(int argc, char ** argv) { // print system information { LOG_TEE("\n"); - LOG_TEE("%s\n", get_system_info(params).c_str()); + LOG_TEE("%s\n", gpt_params_get_system_info(params).c_str()); } const bool add_bos = llama_should_add_bos_token(model); GGML_ASSERT(llama_add_eos_token(model) != 1); @@ -621,8 +621,8 @@ int main(int argc, char ** argv) { if (params.escape) { //process escape sequences, for the initial prompt this is done in common.cpp when we load the params, but for the interactive mode we need to do it here - process_escapes(params.input_prefix); - process_escapes(params.input_suffix); + string_process_escapes(params.input_prefix); + string_process_escapes(params.input_suffix); } suff_rm_leading_spc = params.escape; if (suff_rm_leading_spc && params.input_suffix.find_first_of(' ') == 0 && params.input_suffix.size() > 1) { diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 6bb1f70c3..2afdb3abd 100644 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -200,7 +200,7 @@ static const cmd_params cmd_params_defaults = { /* n_ubatch */ {512}, /* type_k */ {GGML_TYPE_F16}, /* type_v */ {GGML_TYPE_F16}, - /* n_threads */ {get_math_cpu_count()}, + /* n_threads */ {cpu_get_num_math()}, /* n_gpu_layers */ {99}, /* split_mode */ {LLAMA_SPLIT_MODE_LAYER}, /* main_gpu */ {0}, diff --git a/examples/llava/llava-cli.cpp b/examples/llava/llava-cli.cpp index a6d67e5d7..c974900f2 100644 --- a/examples/llava/llava-cli.cpp +++ b/examples/llava/llava-cli.cpp @@ -290,7 +290,7 @@ int main(int argc, char ** argv) { #endif // LOG_DISABLE_LOGS if (params.mmproj.empty() || (params.image.empty() && !prompt_contains_image(params.prompt))) { - gpt_print_usage(argc, argv, params); + gpt_params_print_usage(argc, argv, params); show_additional_info(argc, argv); return 1; } diff --git a/examples/lookahead/lookahead.cpp b/examples/lookahead/lookahead.cpp index 9c3540b20..54f060a85 100644 --- a/examples/lookahead/lookahead.cpp +++ b/examples/lookahead/lookahead.cpp @@ -174,7 +174,7 @@ int main(int argc, char ** argv) { // debug if (dump_kv_cache) { llama_kv_cache_view_update(ctx, &kvc_view); - dump_kv_cache_view_seqs(kvc_view, 40); + llama_kv_cache_dump_view_seqs(kvc_view, 40); } // build the mask from https://lmsys.org/blog/2023-11-21-lookahead-decoding/ diff --git a/examples/lookup/lookup.cpp b/examples/lookup/lookup.cpp index eebbd00a5..83dbee91a 100644 --- a/examples/lookup/lookup.cpp +++ b/examples/lookup/lookup.cpp @@ -121,7 +121,7 @@ int main(int argc, char ** argv){ // debug if (dump_kv_cache) { llama_kv_cache_view_update(ctx, &kvc_view); - dump_kv_cache_view_seqs(kvc_view, 40); + llama_kv_cache_dump_view_seqs(kvc_view, 40); } // print current draft sequence diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 832b51ee0..791dc61a7 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -60,9 +60,9 @@ static void write_logfile( return; } - const std::string timestamp = get_sortable_timestamp(); + const std::string timestamp = string_get_sortable_timestamp(); - const bool success = create_directory_with_parents(params.logdir); + const bool success = fs_create_directory_with_parents(params.logdir); if (!success) { fprintf(stderr, "%s: warning: failed to create logdir %s, cannot write logfile\n", __func__, params.logdir.c_str()); @@ -80,7 +80,7 @@ static void write_logfile( fprintf(logfile, "binary: main\n"); char model_desc[128]; llama_model_desc(model, model_desc, sizeof(model_desc)); - dump_non_result_info_yaml(logfile, params, ctx, timestamp, input_tokens, model_desc); + yaml_dump_non_result_info(logfile, params, ctx, timestamp, input_tokens, model_desc); fprintf(logfile, "\n"); fprintf(logfile, "######################\n"); @@ -88,8 +88,8 @@ static void write_logfile( fprintf(logfile, "######################\n"); fprintf(logfile, "\n"); - dump_string_yaml_multiline(logfile, "output", output.c_str()); - dump_vector_int_yaml(logfile, "output_tokens", output_tokens); + yaml_dump_string_multiline(logfile, "output", output.c_str()); + yaml_dump_vector_int(logfile, "output_tokens", output_tokens); llama_dump_timing_info_yaml(logfile, ctx); fclose(logfile); @@ -181,7 +181,7 @@ int main(int argc, char ** argv) { std::mt19937 rng(params.seed); if (params.random_prompt) { - params.prompt = gpt_random_prompt(rng); + params.prompt = string_random_prompt(rng); } LOG("%s: llama backend init\n", __func__); @@ -219,7 +219,7 @@ int main(int argc, char ** argv) { // print system information { LOG_TEE("\n"); - LOG_TEE("%s\n", get_system_info(params).c_str()); + LOG_TEE("%s\n", gpt_params_get_system_info(params).c_str()); } std::string path_session = params.path_prompt_cache; @@ -879,7 +879,7 @@ int main(int argc, char ** argv) { embd_inp.insert(embd_inp.end(), cml_pfx.begin(), cml_pfx.end()); } if (params.escape) { - process_escapes(buffer); + string_process_escapes(buffer); } const auto line_pfx = ::llama_tokenize(ctx, params.input_prefix, false, true); diff --git a/examples/parallel/parallel.cpp b/examples/parallel/parallel.cpp index 7c5595d6e..c731abb72 100644 --- a/examples/parallel/parallel.cpp +++ b/examples/parallel/parallel.cpp @@ -210,7 +210,7 @@ int main(int argc, char ** argv) { while (true) { if (dump_kv_cache) { llama_kv_cache_view_update(ctx, &kvc_view); - dump_kv_cache_view_seqs(kvc_view, 40); + llama_kv_cache_dump_view_seqs(kvc_view, 40); } llama_batch_clear(batch); diff --git a/examples/perplexity/perplexity.cpp b/examples/perplexity/perplexity.cpp index bae014e6f..30e5e282e 100644 --- a/examples/perplexity/perplexity.cpp +++ b/examples/perplexity/perplexity.cpp @@ -44,9 +44,9 @@ static void write_logfile( return; } - const std::string timestamp = get_sortable_timestamp(); + const std::string timestamp = string_get_sortable_timestamp(); - const bool success = create_directory_with_parents(params.logdir); + const bool success = fs_create_directory_with_parents(params.logdir); if (!success) { fprintf(stderr, "%s: warning: failed to create logdir %s, cannot write logfile\n", __func__, params.logdir.c_str()); @@ -64,7 +64,7 @@ static void write_logfile( fprintf(logfile, "binary: main\n"); char model_desc[128]; llama_model_desc(model, model_desc, sizeof(model_desc)); - dump_non_result_info_yaml(logfile, params, ctx, timestamp, results.tokens, model_desc); + yaml_dump_non_result_info(logfile, params, ctx, timestamp, results.tokens, model_desc); fprintf(logfile, "\n"); fprintf(logfile, "######################\n"); @@ -72,9 +72,9 @@ static void write_logfile( fprintf(logfile, "######################\n"); fprintf(logfile, "\n"); - dump_vector_float_yaml(logfile, "logits", results.logits); + yaml_dump_vector_float(logfile, "logits", results.logits); fprintf(logfile, "ppl_value: %f\n", results.ppl_value); - dump_vector_float_yaml(logfile, "probs", results.probs); + yaml_dump_vector_float(logfile, "probs", results.probs); llama_dump_timing_info_yaml(logfile, ctx); fclose(logfile); @@ -2007,7 +2007,7 @@ int main(int argc, char ** argv) { std::mt19937 rng(params.seed); if (params.random_prompt) { - params.prompt = gpt_random_prompt(rng); + params.prompt = string_random_prompt(rng); } llama_backend_init(); @@ -2035,7 +2035,7 @@ int main(int argc, char ** argv) { // print system information { fprintf(stderr, "\n"); - fprintf(stderr, "%s\n", get_system_info(params).c_str()); + fprintf(stderr, "%s\n", gpt_params_get_system_info(params).c_str()); } struct results_perplexity results; diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index cbb452334..28584e14b 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -259,7 +259,7 @@ int main(int argc, char ** argv) { usage(argv[0]); } } else if (strcmp(argv[arg_idx], "--override-kv") == 0) { - if (arg_idx == argc-1 || !parse_kv_override(argv[++arg_idx], kv_overrides)) { + if (arg_idx == argc-1 || !string_parse_kv_override(argv[++arg_idx], kv_overrides)) { usage(argv[0]); } } else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) { diff --git a/examples/retrieval/retrieval.cpp b/examples/retrieval/retrieval.cpp index 5ba71e76a..4e7530706 100644 --- a/examples/retrieval/retrieval.cpp +++ b/examples/retrieval/retrieval.cpp @@ -11,7 +11,7 @@ struct retrieval_params { }; static void retrieval_params_print_usage(int argc, char ** argv, gpt_params & gpt_params, retrieval_params & params) { - gpt_print_usage(argc, argv, gpt_params); + gpt_params_print_usage(argc, argv, gpt_params); printf("retrieval options:\n"); printf(" --context-file FNAME file containing context to embed.\n"); printf(" specify multiple files by providing --context-file option multiple times.\n"); @@ -226,7 +226,7 @@ int main(int argc, char ** argv) { // print system information { fprintf(stderr, "\n"); - fprintf(stderr, "%s\n", get_system_info(params).c_str()); + fprintf(stderr, "%s\n", gpt_params_get_system_info(params).c_str()); } // max batch size diff --git a/examples/server/server.cpp b/examples/server/server.cpp index 6af5cb96e..e9904263d 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1019,7 +1019,7 @@ struct server_context { sampler_names.emplace_back(sampler_name); } } - slot.sparams.samplers_sequence = sampler_types_from_names(sampler_names, false); + slot.sparams.samplers_sequence = llama_sampling_types_from_names(sampler_names, false); } else { slot.sparams.samplers_sequence = default_sparams.samplers_sequence; } @@ -1256,7 +1256,7 @@ struct server_context { std::vector samplers_sequence; samplers_sequence.reserve(slot.sparams.samplers_sequence.size()); for (const auto & sampler_type : slot.sparams.samplers_sequence) { - samplers_sequence.emplace_back(sampler_type_to_name_string(sampler_type)); + samplers_sequence.emplace_back(llama_sampling_type_to_str(sampler_type)); } return json { @@ -2852,7 +2852,7 @@ static void server_params_parse(int argc, char ** argv, server_params & sparams, invalid_param = true; break; } - if (!parse_kv_override(argv[i], params.kv_overrides)) { + if (!string_parse_kv_override(argv[i], params.kv_overrides)) { fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]); invalid_param = true; break; @@ -3310,7 +3310,7 @@ int main(int argc, char ** argv) { const auto handle_slots_save = [&ctx_server, &res_error, &sparams](const httplib::Request & req, httplib::Response & res, int id_slot) { json request_data = json::parse(req.body); std::string filename = request_data.at("filename"); - if (!validate_file_name(filename)) { + if (!fs_validate_filename(filename)) { res_error(res, format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST)); return; } @@ -3340,7 +3340,7 @@ int main(int argc, char ** argv) { const auto handle_slots_restore = [&ctx_server, &res_error, &sparams](const httplib::Request & req, httplib::Response & res, int id_slot) { json request_data = json::parse(req.body); std::string filename = request_data.at("filename"); - if (!validate_file_name(filename)) { + if (!fs_validate_filename(filename)) { res_error(res, format_error_response("Invalid filename", ERROR_TYPE_INVALID_REQUEST)); return; } From 197ff91462dd05bb9a3be03578114abf0c355536 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 22 May 2024 20:05:38 +0300 Subject: [PATCH 109/137] build : remove zig (#7471) --- .github/workflows/zig-build.yml | 29 ------ build.zig | 172 -------------------------------- 2 files changed, 201 deletions(-) delete mode 100644 .github/workflows/zig-build.yml delete mode 100644 build.zig diff --git a/.github/workflows/zig-build.yml b/.github/workflows/zig-build.yml deleted file mode 100644 index 747c35cc0..000000000 --- a/.github/workflows/zig-build.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Zig CI - -on: - pull_request: - push: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} - cancel-in-progress: true - -jobs: - build: - strategy: - fail-fast: false - matrix: - runs-on: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - uses: goto-bus-stop/setup-zig@v2 - with: - version: 0.11.0 - - name: Build Summary - run: zig build --summary all -freference-trace diff --git a/build.zig b/build.zig deleted file mode 100644 index 267c976b1..000000000 --- a/build.zig +++ /dev/null @@ -1,172 +0,0 @@ -// Compatible with Zig Version 0.11.0 -const std = @import("std"); -const ArrayList = std.ArrayList; -const Compile = std.Build.Step.Compile; -const ConfigHeader = std.Build.Step.ConfigHeader; -const Mode = std.builtin.Mode; -const CrossTarget = std.zig.CrossTarget; - -const Maker = struct { - builder: *std.build.Builder, - target: CrossTarget, - optimize: Mode, - enable_lto: bool, - - include_dirs: ArrayList([]const u8), - cflags: ArrayList([]const u8), - cxxflags: ArrayList([]const u8), - objs: ArrayList(*Compile), - - fn addInclude(m: *Maker, dir: []const u8) !void { - try m.include_dirs.append(dir); - } - fn addProjectInclude(m: *Maker, path: []const []const u8) !void { - try m.addInclude(try m.builder.build_root.join(m.builder.allocator, path)); - } - fn addCFlag(m: *Maker, flag: []const u8) !void { - try m.cflags.append(flag); - } - fn addCxxFlag(m: *Maker, flag: []const u8) !void { - try m.cxxflags.append(flag); - } - fn addFlag(m: *Maker, flag: []const u8) !void { - try m.addCFlag(flag); - try m.addCxxFlag(flag); - } - - fn init(builder: *std.build.Builder) !Maker { - const target = builder.standardTargetOptions(.{}); - const zig_version = @import("builtin").zig_version_string; - const commit_hash = try std.ChildProcess.exec( - .{ .allocator = builder.allocator, .argv = &.{ "git", "rev-parse", "HEAD" } }, - ); - try std.fs.cwd().writeFile("common/build-info.cpp", builder.fmt( - \\int LLAMA_BUILD_NUMBER = {}; - \\char const *LLAMA_COMMIT = "{s}"; - \\char const *LLAMA_COMPILER = "Zig {s}"; - \\char const *LLAMA_BUILD_TARGET = "{s}"; - \\ - , .{ 0, commit_hash.stdout[0 .. commit_hash.stdout.len - 1], zig_version, try target.allocDescription(builder.allocator) })); - var m = Maker{ - .builder = builder, - .target = target, - .optimize = builder.standardOptimizeOption(.{}), - .enable_lto = false, - .include_dirs = ArrayList([]const u8).init(builder.allocator), - .cflags = ArrayList([]const u8).init(builder.allocator), - .cxxflags = ArrayList([]const u8).init(builder.allocator), - .objs = ArrayList(*Compile).init(builder.allocator), - }; - - try m.addCFlag("-std=c11"); - try m.addCxxFlag("-std=c++11"); - try m.addProjectInclude(&.{}); - try m.addProjectInclude(&.{"common"}); - return m; - } - - fn obj(m: *const Maker, name: []const u8, src: []const u8) *Compile { - const o = m.builder.addObject(.{ .name = name, .target = m.target, .optimize = m.optimize }); - if (o.target.getAbi() != .msvc) - o.defineCMacro("_GNU_SOURCE", null); - - if (std.mem.endsWith(u8, src, ".c")) { - o.addCSourceFiles(&.{src}, m.cflags.items); - o.linkLibC(); - } else { - o.addCSourceFiles(&.{src}, m.cxxflags.items); - if (o.target.getAbi() == .msvc) { - o.linkLibC(); // need winsdk + crt - } else { - // linkLibCpp already add (libc++ + libunwind + libc) - o.linkLibCpp(); - } - } - for (m.include_dirs.items) |i| o.addIncludePath(.{ .path = i }); - o.want_lto = m.enable_lto; - return o; - } - - fn exe(m: *const Maker, name: []const u8, src: []const u8, deps: []const *Compile) *Compile { - const e = m.builder.addExecutable(.{ .name = name, .target = m.target, .optimize = m.optimize }); - e.addCSourceFiles(&.{src}, m.cxxflags.items); - for (deps) |d| e.addObject(d); - for (m.objs.items) |o| e.addObject(o); - for (m.include_dirs.items) |i| e.addIncludePath(.{ .path = i }); - - // https://github.com/ziglang/zig/issues/15448 - if (e.target.getAbi() == .msvc) { - e.linkLibC(); // need winsdk + crt - } else { - // linkLibCpp already add (libc++ + libunwind + libc) - e.linkLibCpp(); - } - m.builder.installArtifact(e); - e.want_lto = m.enable_lto; - return e; - } -}; - -pub fn build(b: *std.build.Builder) !void { - var make = try Maker.init(b); - make.enable_lto = b.option(bool, "lto", "Enable LTO optimization, (default: false)") orelse false; - - const ggml = make.obj("ggml", "ggml.c"); - const sgemm = make.obj("sgemm", "sgemm.cpp"); - const ggml_alloc = make.obj("ggml-alloc", "ggml-alloc.c"); - const ggml_backend = make.obj("ggml-backend", "ggml-backend.c"); - const ggml_quants = make.obj("ggml-quants", "ggml-quants.c"); - const unicode = make.obj("unicode", "unicode.cpp"); - const unicode_data = make.obj("unicode-data", "unicode-data.cpp"); - const llama = make.obj("llama", "llama.cpp"); - const buildinfo = make.obj("common", "common/build-info.cpp"); - const common = make.obj("common", "common/common.cpp"); - const console = make.obj("console", "common/console.cpp"); - const sampling = make.obj("sampling", "common/sampling.cpp"); - const grammar_parser = make.obj("grammar-parser", "common/grammar-parser.cpp"); - const json_schema_to_grammar = make.obj("json-schema-to-grammar", "common/json-schema-to-grammar.cpp"); - const train = make.obj("train", "common/train.cpp"); - const clip = make.obj("clip", "examples/llava/clip.cpp"); - const llava = make.obj("llava", "examples/llava/llava.cpp"); - - _ = make.exe("main", "examples/main/main.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo, console, grammar_parser }); - _ = make.exe("quantize", "examples/quantize/quantize.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo }); - _ = make.exe("perplexity", "examples/perplexity/perplexity.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo }); - _ = make.exe("embedding", "examples/embedding/embedding.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo }); - _ = make.exe("finetune", "examples/finetune/finetune.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo, train }); - _ = make.exe("train-text-from-scratch", "examples/train-text-from-scratch/train-text-from-scratch.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, json_schema_to_grammar, buildinfo, train }); - - const server = make.exe("server", "examples/server/server.cpp", &.{ ggml, sgemm, ggml_alloc, ggml_backend, ggml_quants, llama, unicode, unicode_data, common, sampling, json_schema_to_grammar, buildinfo, grammar_parser, clip, llava }); - if (server.target.isWindows()) { - server.linkSystemLibrary("ws2_32"); - } - - const server_assets = [_][]const u8{ "index.html", "index.js", "completion.js", "json-schema-to-grammar.mjs" }; - for (server_assets) |asset| { - const input_path = b.fmt("examples/server/public/{s}", .{asset}); - const output_path = b.fmt("examples/server/{s}.hpp", .{asset}); - - // Portable equivalent of `b.addSystemCommand(&.{ "xxd", "-n", asset, "-i", input_path, output_path }) })`: - - const input = try std.fs.cwd().readFileAlloc(b.allocator, input_path, std.math.maxInt(usize)); - defer b.allocator.free(input); - - var buf = std.ArrayList(u8).init(b.allocator); - defer buf.deinit(); - - for (input) |byte| { - try std.fmt.format(buf.writer(), "0x{X:0>2}, ", .{byte}); - } - - var name = try std.mem.replaceOwned(u8, b.allocator, asset, "-", "_"); - defer b.allocator.free(name); - std.mem.replaceScalar(u8, name, '.', '_'); - - try std.fs.cwd().writeFile(output_path, b.fmt( - "unsigned char {s}[] = {{{s}}};\nunsigned int {s}_len = {d};\n", - .{ name, buf.items, name, input.len }, - )); - - std.debug.print("Dumped hex of \"{s}\" ({s}) to {s}\n", .{ input_path, name, output_path }); - } -} From 1e374365d170b7f692fd7753c145e21bc14486c8 Mon Sep 17 00:00:00 2001 From: HanishKVC Date: Wed, 22 May 2024 23:23:21 +0530 Subject: [PATCH 110/137] SimpleChat: a simple and dumb web front end for testing /chat/completions and /completions end points and try chat (#7350) * SimpleChat: Add a skeletal html page Contains a div placeholder for showing chat messages till now a text-input for allowing user to enter next chat message/query to the model. a submit button to allow sending of the user entered message and chat till now to the model. * SimpleChat: A js skeleton with SimpleChat class Allows maintaining an array of chat message. Allows adding chat message (from any of the roles be it system, user, assistant, ...) Allows showing chat messages till now, in a given div element. * SimpleChat: request_json, globals, startme * SimpleChatJS: Roles Class, submitClick Define Role class with static members corresponding to the roles. Update startme to * Get hold of the ui elements. * Attach a click handler to submit button, which adds the user input to xchats array and shows the chat messages till now in chat div element. Trap DOMContentLoaded to trigger startme * SimpleChat:HTML: Bring in the js file * SimpleChat: Rather value wrt input text element * SimpleChat: Also add completions related prompt * SimpleChat: Use common helper logic wrt json data * SimpleChat: Move handling of submit request into its own func * SimpleChat: Try handshake with llm over its web service endpoint * SimpleChat:JS: Extract model response and show to user * SimpleChat:JS: Messages/Prompt, indicate working to end user * SimpleChat: Try keep input element in view * SimpleChat: Diff user/assistant msgs, Make input wider Also show a default message to user Also add some metas * SimpleChat: Move into its own sub directory to avoid confusion * SimpleChat:sh: Add simple shell script to run python3 http.server So one needs to run the llm server locally then run this script and access it using a local browser * SimpleChat:JS: Try trap enter key press wrt input text field So user can either press submit button or press enter key * SimpleChat: Allow user to select chat or completion mode * SimpleChat: Dont submit if already submitted and waiting Also make chat the default selection wrt mode * SimpleChat:JS: Handle difference in response Try read the assistance response from appropriate field in the response got. Also examples/server seems to return the response in a slightly different field, so try account for that also. * SimpleChat:JS: Force completion mode be single message by default * SimpleChat: Add a simple readme file * SimpleChat:HTML: Cleanup/structure UI a bit, Add input for system * SimpleChat:Allow system prompt to be set, if provided before user * SimpleChat: Ignore empty user input, without trimming * SimpleChat:Alert user if they provide sysprompt late or change it * SimpleChat: Move handling systemprompt into its own func * SimpleChat:HTML: Add a style for system role message * SimpleChat: Update the readme file * SimpleChat:CSS: Move style info into its own css file To keep it simple, clean and seperate so that things are not unnecessarily cluttered. * SimpleChat:CSS: Allow for chat div to be scrollable * SimpleChat:JS: Try ensure the last entry in chat is visible Needed because now only the chat div is scrollable and not the full page. In last commit the chat div size was fixed to 75% vertical height, so the full page no longer scrolls, so the old bring user-input element to view wont work, instead now the last element in the chat div should be brought into view. * SimpleChat:JS: bottom of element visible, Set focus to user input As the generated text could be multiple lines and occupy more space that the full scrollable div's vertical space, make the bottom of the last element (which can be such a generated text) in the div visible by scrolling. Ensure that the user input box has focus * SimpleChat: Update notes a bit. Try keep browser happy Avoid browser quirk mode with DOCTYPE. Help with accessibility a bit by specifying the language explicitly. Specify the char encoding explicitly, inturn utf-8 is a safe bet, even with intermixing of languages if reqd in future. Add a cache-control http-equiv meta tag, which in all probability will be ignored. Defer js loading and execution, just for fun and future, not that critical here as it stands now. * SimpleChat:HTML:Group user input+btn together; Note about multichat * SimpleChat:JS: Allow for changing system prompt anytime for future * SimpleChat:Readme: Note about handle_systemprompt begin/anytime * SimpleChat:HTML: Add viewport meta for better mobile friendliness Without this the page content may look too small. * SimpleChat:HtmlCss: Cleanup UI flow set margin wrt vmin rather than vw or vh so portrait/landscape ok. Use flex and flex-grow to put things on the same line as well as distribute available space as needed. Given two main elements/line so it remains simple. In each line have one element with grows and one sits with a basic comfortably fixed size. * SimpleChat: textarea for multiline user chat, inturn shift+enter 4 enter * SimpleChat: Make vertical layout better responsive (flex based) Also needed to make things cleaner and properly usable whether landscape or portrait, after changing to multiline textarea rather than single line user input. Avoid hardcoding the chat-till-now display area height, instead make it a flex-growable within a flex column of ui elements within a fixed vertical area. * SimpleChat: Rename simplechat.html to index.html, update readme Instead of providing a seperate shell script, update the readme wrt how to run/use this web front end. * SimpleChat: Screen fixed view and scrolling, Printing full * SimpleChat:JS:CI: Avoid space at end of jsdoc param line * SimpleChat:JS: MultiChat initial skeleton Will help maintain multiple independent chats in future * SimpleChat:JS: Move system prompt begin/anytime into SimpleChat * SimpleChat:JS:Keep MultiChatUI simple for now Worry about different chats with different servers for later. * SimpleChat:JS: Move handle submit into MultiChat, build on same Create an instance of MultiChatUI and inturn a instance of chat session, which is what the UI will inturn work on. * SimpleChat:JS: Move to dictionary of SimpleChat, instead of array * SimpleChat: Move ui elements into MultiChatUI, Update el IDs Move ui elements into MultiChatUI, so that current handleUserSubmit doesnt need to take the element arguments. Also in future, when user is allowed to switch between different chat sessions, the UI can be updated as needed by using the elements in UI already known to MultiChatUI instance. Rename the element ids' so that they follow a common convention, as well as one can identify what the element represents in a more consistant manner. * SimpleChat:MCUI:Show available chat sessions, try switch btw them Previous commits brought in / consolidated existing logic into MultiChatUI class. Now start adding logic towards multichat support * show buttons indicating available chat sessions * on sessin button click, try switch to that session * SimpleChat:MCUI: Store and use current chat session id Also allow to switch chat session optionally, wrt some of the related helpers. setup for two chat sessions by default. * SimpleChat:MCUI: Delay enabling user-input to avoid race Re-enable user-input, only after response to a user query has been updated to the chat-div. This ensures that if user tries to switch chat session, it wont be allowed till chat-request-response flow is done. * SimpleChat: Take care of system prompt Helper to get the latest system prompt and inturn use same to set the system prompt ui, when switching. Ensure that system prompt is set if and when enter key is pressed. * SimpleChat:GetSystemLatest, fix a oversight. * SimpleChat:MCUI: Allow selected chat-session btn to be highlighted Also have a general helper for setting class of children. * SimpleChat:Cleanup corners Show system prompt in chat space, when it is set by pressing enter, as a feedback to user. Alert user, if they try to switch chat session in the middle of waiting for a response from the ai model. * SimpleChat:MCUI: Ensure req-resp failure doesnt lock up things * SimpleChat:MCUI: Support for new chat sessions Also a general create button helper. * SimpleChat:MCUI: CreateSessionBtn helper, use wrt NewChat Also fix a oversight wrt using stale data wrt the list of chat sessions. * SimpleChat:MCUI: NewChat btn first before existing chat sessions * SimpleChat:MCUI:CornerCases:Skip new chat, show only if current Skip NewChat if user cancels or if one waiting for response from the ai model. Dont show a chat with newly got ai model response, if current chat session has changed, some how. Chat session shouldnt be allowed to change, if there is a pending response, but still as a additional sanity check. * SimpleChat: Update readme, title, show usage if no chat to show * SimpleChat: Cleanup the log/dialog messages a bit --- examples/server/public_simplechat/index.html | 52 ++ examples/server/public_simplechat/readme.md | 81 +++ .../server/public_simplechat/simplechat.css | 61 +++ .../server/public_simplechat/simplechat.js | 478 ++++++++++++++++++ 4 files changed, 672 insertions(+) create mode 100644 examples/server/public_simplechat/index.html create mode 100644 examples/server/public_simplechat/readme.md create mode 100644 examples/server/public_simplechat/simplechat.css create mode 100644 examples/server/public_simplechat/simplechat.js diff --git a/examples/server/public_simplechat/index.html b/examples/server/public_simplechat/index.html new file mode 100644 index 000000000..1eb390b85 --- /dev/null +++ b/examples/server/public_simplechat/index.html @@ -0,0 +1,52 @@ + + + + SimpleChat (LlamaCPP, ...) + + + + + + + + + + +
+ +
+

SimpleChat

+
+ + +
+
+ +
+ +
+
+ + +
+ +
+
+

Enter the system prompt above, before entering/submitting any user query.

+

Enter your text to the ai assistant below.

+

Use shift+enter for inserting enter.

+

Refresh the page to start over fresh.

+
+ +
+
+ + +
+ +
+ + diff --git a/examples/server/public_simplechat/readme.md b/examples/server/public_simplechat/readme.md new file mode 100644 index 000000000..5ac8258f2 --- /dev/null +++ b/examples/server/public_simplechat/readme.md @@ -0,0 +1,81 @@ + +# SimpleChat + +by Humans for All. + + +## overview + +This simple web frontend, allows triggering/testing the server's /completions or /chat/completions endpoints +in a simple way with minimal code from a common code base. Inturn additionally it tries to allow single or +multiple independent back and forth chatting to an extent, with the ai llm model at a basic level, with their +own system prompts. + +The UI follows a responsive web design so that the layout can adapt to available display space in a usable +enough manner, in general. + +NOTE: Given that the idea is for basic minimal testing, it doesnt bother with any model context length and +culling of old messages from the chat. + +NOTE: It doesnt set any parameters other than temperature for now. However if someone wants they can update +the js file as needed. + + +## usage + +One could run this web frontend directly using server itself or if anyone is thinking of adding a built in web +frontend to configure the server over http(s) or so, then run this web frontend using something like python's +http module. + +### running using examples/server + +bin/server -m path/model.gguf --path ../examples/server/public_simplechat [--port PORT] + +### running using python3's server module + +first run examples/server +* bin/server -m path/model.gguf + +next run this web front end in examples/server/public_simplechat +* cd ../examples/server/public_simplechat +* python3 -m http.server PORT + +### using the front end + +Open this simple web front end from your local browser +* http://127.0.0.1:PORT/index.html + +Once inside +* Select between chat and completion mode. By default it is set to chat mode. +* If you want to provide a system prompt, then ideally enter it first, before entering any user query. + * if chat.add_system_begin is used + * you cant change the system prompt, after it is has been submitted once along with user query. + * you cant set a system prompt, after you have submitted any user query + * if chat.add_system_anytime is used + * one can change the system prompt any time during chat, by changing the contents of system prompt. + * inturn the updated/changed system prompt will be inserted into the chat session. + * this allows for the subsequent user chatting to be driven by the new system prompt set above. +* Enter your query and either press enter or click on the submit button. + If you want to insert enter (\n) as part of your chat/query to ai model, use shift+enter. +* Wait for the logic to communicate with the server and get the response. + * the user is not allowed to enter any fresh query during this time. + * the user input box will be disabled and a working message will be shown in it. +* just refresh the page, to reset wrt the chat history and or system prompt and start afresh. +* Using NewChat one can start independent chat sessions. + * two independent chat sessions are setup by default. + + +## Devel note + +Sometimes the browser may be stuborn with caching of the file, so your updates to html/css/js +may not be visible. Also remember that just refreshing/reloading page in browser or for that +matter clearing site data, dont directly override site caching in all cases. Worst case you may +have to change port. Or in dev tools of browser, you may be able to disable caching fully. + +Concept of multiple chat sessions with different servers, as well as saving and restoring of +those across browser usage sessions, can be woven around the SimpleChat/MultiChatUI class and +its instances relatively easily, however given the current goal of keeping this simple, it has +not been added, for now. + +By switching between chat.add_system_begin/anytime, one can control whether one can change +the system prompt, anytime during the conversation or only at the beginning. diff --git a/examples/server/public_simplechat/simplechat.css b/examples/server/public_simplechat/simplechat.css new file mode 100644 index 000000000..d45f50a95 --- /dev/null +++ b/examples/server/public_simplechat/simplechat.css @@ -0,0 +1,61 @@ +/** + * the styling of the simplechat web frontend + * by Humans for All + */ + +#fullbody { + height: 98vh; +} + +.heading { + background-color: lightgray; +} + +.session-selected { + background-color: lightblue; +} + +.role-system { + background-color: lightblue; +} +.role-user { + background-color: lightgray; +} + +.flex-grow { + flex-grow: 1; +} +.float-right { + float: right; +} + +#chat-div { + overflow: scroll; + flex-grow: 1; + flex-shrink: 1; + min-height: 40vh; +} +button { + min-width: 8vw; +} + +.sameline { + display: flex; + flex-direction: row; +} +.samecolumn { + display: flex; + flex-direction: column; +} + +* { + margin: 0.6vmin; +} + +@media print { + + #fullbody { + height: auto; + } + +} diff --git a/examples/server/public_simplechat/simplechat.js b/examples/server/public_simplechat/simplechat.js new file mode 100644 index 000000000..3fc4dbc20 --- /dev/null +++ b/examples/server/public_simplechat/simplechat.js @@ -0,0 +1,478 @@ +// @ts-check +// A simple completions and chat/completions test related web front end logic +// by Humans for All + +class Roles { + static System = "system"; + static User = "user"; + static Assistant = "assistant"; +} + +class ApiEP { + static Chat = "chat"; + static Completion = "completion"; +} + +let gUsageMsg = ` +

Enter the system prompt above, before entering/submitting any user query.

+

Enter your text to the ai assistant below.

+

Use shift+enter for inserting enter.

+

Refresh the page to start over fresh.

+`; + +class SimpleChat { + + constructor() { + /** + * Maintain in a form suitable for common LLM web service chat/completions' messages entry + * @type {{role: string, content: string}[]} + */ + this.xchat = []; + this.iLastSys = -1; + } + + /** + * Add an entry into xchat + * @param {string} role + * @param {string|undefined|null} content + */ + add(role, content) { + if ((content == undefined) || (content == null) || (content == "")) { + return false; + } + this.xchat.push( {role: role, content: content} ); + if (role == Roles.System) { + this.iLastSys = this.xchat.length - 1; + } + return true; + } + + /** + * Show the contents in the specified div + * @param {HTMLDivElement} div + * @param {boolean} bClear + */ + show(div, bClear=true) { + if (bClear) { + div.replaceChildren(); + } + let last = undefined; + for(const x of this.xchat) { + let entry = document.createElement("p"); + entry.className = `role-${x.role}`; + entry.innerText = `${x.role}: ${x.content}`; + div.appendChild(entry); + last = entry; + } + if (last !== undefined) { + last.scrollIntoView(false); + } else { + if (bClear) { + div.innerHTML = gUsageMsg; + } + } + } + + /** + * Add needed fields wrt json object to be sent wrt LLM web services completions endpoint + * Convert the json into string. + * @param {Object} obj + */ + request_jsonstr(obj) { + obj["temperature"] = 0.7; + return JSON.stringify(obj); + } + + /** + * Return a string form of json object suitable for chat/completions + */ + request_messages_jsonstr() { + let req = { + messages: this.xchat, + } + return this.request_jsonstr(req); + } + + /** + * Return a string form of json object suitable for /completions + */ + request_prompt_jsonstr() { + let prompt = ""; + for(const chat of this.xchat) { + prompt += `${chat.role}: ${chat.content}\n`; + } + let req = { + prompt: prompt, + } + return this.request_jsonstr(req); + } + + /** + * Allow setting of system prompt, but only at begining. + * @param {string} sysPrompt + * @param {string} msgTag + */ + add_system_begin(sysPrompt, msgTag) { + if (this.xchat.length == 0) { + if (sysPrompt.length > 0) { + return this.add(Roles.System, sysPrompt); + } + } else { + if (sysPrompt.length > 0) { + if (this.xchat[0].role !== Roles.System) { + console.error(`ERRR:SimpleChat:SC:${msgTag}:You need to specify system prompt before any user query, ignoring...`); + } else { + if (this.xchat[0].content !== sysPrompt) { + console.error(`ERRR:SimpleChat:SC:${msgTag}:You cant change system prompt, mid way through, ignoring...`); + } + } + } + } + return false; + } + + /** + * Allow setting of system prompt, at any time. + * @param {string} sysPrompt + * @param {string} msgTag + */ + add_system_anytime(sysPrompt, msgTag) { + if (sysPrompt.length <= 0) { + return false; + } + + if (this.iLastSys < 0) { + return this.add(Roles.System, sysPrompt); + } + + let lastSys = this.xchat[this.iLastSys].content; + if (lastSys !== sysPrompt) { + return this.add(Roles.System, sysPrompt); + } + return false; + } + + /** + * Retrieve the latest system prompt. + */ + get_system_latest() { + if (this.iLastSys == -1) { + return ""; + } + let sysPrompt = this.xchat[this.iLastSys].content; + return sysPrompt; + } + +} + + +let gBaseURL = "http://127.0.0.1:8080"; +let gChatURL = { + 'chat': `${gBaseURL}/chat/completions`, + 'completion': `${gBaseURL}/completions`, +} +const gbCompletionFreshChatAlways = true; + + +/** + * Set the class of the children, based on whether it is the idSelected or not. + * @param {HTMLDivElement} elBase + * @param {string} idSelected + * @param {string} classSelected + * @param {string} classUnSelected + */ +function el_children_config_class(elBase, idSelected, classSelected, classUnSelected="") { + for(let child of elBase.children) { + if (child.id == idSelected) { + child.className = classSelected; + } else { + child.className = classUnSelected; + } + } +} + +/** + * Create button and set it up. + * @param {string} id + * @param {(this: HTMLButtonElement, ev: MouseEvent) => any} callback + * @param {string | undefined} name + * @param {string | undefined} innerText + */ +function el_create_button(id, callback, name=undefined, innerText=undefined) { + if (!name) { + name = id; + } + if (!innerText) { + innerText = id; + } + let btn = document.createElement("button"); + btn.id = id; + btn.name = name; + btn.innerText = innerText; + btn.addEventListener("click", callback); + return btn; +} + + +class MultiChatUI { + + constructor() { + /** @type {Object} */ + this.simpleChats = {}; + /** @type {string} */ + this.curChatId = ""; + + // the ui elements + this.elInSystem = /** @type{HTMLInputElement} */(document.getElementById("system-in")); + this.elDivChat = /** @type{HTMLDivElement} */(document.getElementById("chat-div")); + this.elBtnUser = /** @type{HTMLButtonElement} */(document.getElementById("user-btn")); + this.elInUser = /** @type{HTMLInputElement} */(document.getElementById("user-in")); + this.elSelectApiEP = /** @type{HTMLSelectElement} */(document.getElementById("api-ep")); + this.elDivSessions = /** @type{HTMLDivElement} */(document.getElementById("sessions-div")); + + this.validate_element(this.elInSystem, "system-in"); + this.validate_element(this.elDivChat, "chat-div"); + this.validate_element(this.elInUser, "user-in"); + this.validate_element(this.elSelectApiEP, "api-ep"); + this.validate_element(this.elDivChat, "sessions-div"); + } + + /** + * Check if the element got + * @param {HTMLElement | null} el + * @param {string} msgTag + */ + validate_element(el, msgTag) { + if (el == null) { + throw Error(`ERRR:SimpleChat:MCUI:${msgTag} element missing in html...`); + } else { + console.debug(`INFO:SimpleChat:MCUI:${msgTag} Id[${el.id}] Name[${el["name"]}]`); + } + } + + /** + * Reset user input ui. + * * clear user input + * * enable user input + * * set focus to user input + */ + ui_reset_userinput() { + this.elInUser.value = ""; + this.elInUser.disabled = false; + this.elInUser.focus(); + } + + /** + * Setup the needed callbacks wrt UI, curChatId to defaultChatId and + * optionally switch to specified defaultChatId. + * @param {string} defaultChatId + * @param {boolean} bSwitchSession + */ + setup_ui(defaultChatId, bSwitchSession=false) { + + this.curChatId = defaultChatId; + if (bSwitchSession) { + this.handle_session_switch(this.curChatId); + } + + this.elBtnUser.addEventListener("click", (ev)=>{ + if (this.elInUser.disabled) { + return; + } + this.handle_user_submit(this.curChatId, this.elSelectApiEP.value).catch((/** @type{Error} */reason)=>{ + let msg = `ERRR:SimpleChat\nMCUI:HandleUserSubmit:${this.curChatId}\n${reason.name}:${reason.message}`; + console.debug(msg.replace("\n", ":")); + alert(msg); + this.ui_reset_userinput(); + }); + }); + + this.elInUser.addEventListener("keyup", (ev)=> { + // allow user to insert enter into their message using shift+enter. + // while just pressing enter key will lead to submitting. + if ((ev.key === "Enter") && (!ev.shiftKey)) { + this.elBtnUser.click(); + ev.preventDefault(); + } + }); + + this.elInSystem.addEventListener("keyup", (ev)=> { + // allow user to insert enter into the system prompt using shift+enter. + // while just pressing enter key will lead to setting the system prompt. + if ((ev.key === "Enter") && (!ev.shiftKey)) { + let chat = this.simpleChats[this.curChatId]; + chat.add_system_anytime(this.elInSystem.value, this.curChatId); + chat.show(this.elDivChat); + ev.preventDefault(); + } + }); + + } + + /** + * Setup a new chat session and optionally switch to it. + * @param {string} chatId + * @param {boolean} bSwitchSession + */ + new_chat_session(chatId, bSwitchSession=false) { + this.simpleChats[chatId] = new SimpleChat(); + if (bSwitchSession) { + this.handle_session_switch(chatId); + } + } + + /** + * Handle user query submit request, wrt specified chat session. + * @param {string} chatId + * @param {string} apiEP + */ + async handle_user_submit(chatId, apiEP) { + + let chat = this.simpleChats[chatId]; + + chat.add_system_anytime(this.elInSystem.value, chatId); + + let content = this.elInUser.value; + if (!chat.add(Roles.User, content)) { + console.debug(`WARN:SimpleChat:MCUI:${chatId}:HandleUserSubmit:Ignoring empty user input...`); + return; + } + chat.show(this.elDivChat); + + let theBody; + let theUrl = gChatURL[apiEP] + if (apiEP == ApiEP.Chat) { + theBody = chat.request_messages_jsonstr(); + } else { + theBody = chat.request_prompt_jsonstr(); + } + + this.elInUser.value = "working..."; + this.elInUser.disabled = true; + console.debug(`DBUG:SimpleChat:MCUI:${chatId}:HandleUserSubmit:${theUrl}:ReqBody:${theBody}`); + let resp = await fetch(theUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: theBody, + }); + + let respBody = await resp.json(); + console.debug(`DBUG:SimpleChat:MCUI:${chatId}:HandleUserSubmit:RespBody:${JSON.stringify(respBody)}`); + let assistantMsg; + if (apiEP == ApiEP.Chat) { + assistantMsg = respBody["choices"][0]["message"]["content"]; + } else { + try { + assistantMsg = respBody["choices"][0]["text"]; + } catch { + assistantMsg = respBody["content"]; + } + } + chat.add(Roles.Assistant, assistantMsg); + if (chatId == this.curChatId) { + chat.show(this.elDivChat); + } else { + console.debug(`DBUG:SimpleChat:MCUI:HandleUserSubmit:ChatId has changed:[${chatId}] [${this.curChatId}]`); + } + // Purposefully clear at end rather than begin of this function + // so that one can switch from chat to completion mode and sequece + // in a completion mode with multiple user-assistant chat data + // from before to be sent/occur once. + if ((apiEP == ApiEP.Completion) && (gbCompletionFreshChatAlways)) { + chat.xchat.length = 0; + } + this.ui_reset_userinput(); + } + + /** + * Show buttons for NewChat and available chat sessions, in the passed elDiv. + * If elDiv is undefined/null, then use this.elDivSessions. + * Take care of highlighting the selected chat-session's btn. + * @param {HTMLDivElement | undefined} elDiv + */ + show_sessions(elDiv=undefined) { + if (!elDiv) { + elDiv = this.elDivSessions; + } + elDiv.replaceChildren(); + // Btn for creating new chat session + let btnNew = el_create_button("New CHAT", (ev)=> { + if (this.elInUser.disabled) { + console.error(`ERRR:SimpleChat:MCUI:NewChat:Current session [${this.curChatId}] awaiting response, ignoring request...`); + alert("ERRR:SimpleChat\nMCUI:NewChat\nWait for response to pending query, before starting new chat session"); + return; + } + let chatId = `Chat${Object.keys(this.simpleChats).length}`; + let chatIdGot = prompt("INFO:SimpleChat\nMCUI:NewChat\nEnter id for new chat session", chatId); + if (!chatIdGot) { + console.error("ERRR:SimpleChat:MCUI:NewChat:Skipping based on user request..."); + return; + } + this.new_chat_session(chatIdGot, true); + this.create_session_btn(elDiv, chatIdGot); + el_children_config_class(elDiv, chatIdGot, "session-selected", ""); + }); + elDiv.appendChild(btnNew); + // Btns for existing chat sessions + let chatIds = Object.keys(this.simpleChats); + for(let cid of chatIds) { + let btn = this.create_session_btn(elDiv, cid); + if (cid == this.curChatId) { + btn.className = "session-selected"; + } + } + } + + create_session_btn(elDiv, cid) { + let btn = el_create_button(cid, (ev)=>{ + let target = /** @type{HTMLButtonElement} */(ev.target); + console.debug(`DBUG:SimpleChat:MCUI:SessionClick:${target.id}`); + if (this.elInUser.disabled) { + console.error(`ERRR:SimpleChat:MCUI:SessionClick:${target.id}:Current session [${this.curChatId}] awaiting response, ignoring switch...`); + alert("ERRR:SimpleChat\nMCUI:SessionClick\nWait for response to pending query, before switching"); + return; + } + this.handle_session_switch(target.id); + el_children_config_class(elDiv, target.id, "session-selected", ""); + }); + elDiv.appendChild(btn); + return btn; + } + + /** + * Switch ui to the specified chatId and set curChatId to same. + * @param {string} chatId + */ + async handle_session_switch(chatId) { + let chat = this.simpleChats[chatId]; + if (chat == undefined) { + console.error(`ERRR:SimpleChat:MCUI:HandleSessionSwitch:${chatId} missing...`); + return; + } + this.elInSystem.value = chat.get_system_latest(); + this.elInUser.value = ""; + chat.show(this.elDivChat); + this.elInUser.focus(); + this.curChatId = chatId; + console.log(`INFO:SimpleChat:MCUI:HandleSessionSwitch:${chatId} entered...`); + } + +} + + +let gMuitChat; +const gChatIds = [ "Default", "Other" ]; + +function startme() { + console.log("INFO:SimpleChat:StartMe:Starting..."); + gMuitChat = new MultiChatUI(); + for (let cid of gChatIds) { + gMuitChat.new_chat_session(cid); + } + gMuitChat.setup_ui(gChatIds[0]); + gMuitChat.show_sessions(); +} + +document.addEventListener("DOMContentLoaded", startme); From cd93a28cb1446319af5e2f4b416174c3a8e43546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20G=C3=A4=C3=9Fler?= Date: Thu, 23 May 2024 00:31:20 +0200 Subject: [PATCH 111/137] CUDA: fix FA out-of-bounds reads (#7479) --- ggml-cuda/fattn-tile-f16.cu | 2 +- ggml-cuda/fattn-tile-f32.cu | 2 +- ggml-cuda/fattn-vec-f16.cu | 6 +++--- ggml-cuda/fattn-vec-f32.cu | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ggml-cuda/fattn-tile-f16.cu b/ggml-cuda/fattn-tile-f16.cu index 586d469c0..cdb5eaff7 100644 --- a/ggml-cuda/fattn-tile-f16.cu +++ b/ggml-cuda/fattn-tile-f16.cu @@ -83,7 +83,7 @@ static __global__ void flash_attn_tile_ext_f16( for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { const int i = i0 + threadIdx.x; - const float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i]; + const float2 tmp = ic0 + j < ne01 ? Q_f2[j*(nb01/sizeof(float2)) + i] : make_float2(0.0f, 0.0f); Q_h2[j][i] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y); } } diff --git a/ggml-cuda/fattn-tile-f32.cu b/ggml-cuda/fattn-tile-f32.cu index b6ef8eb48..5a3de2918 100644 --- a/ggml-cuda/fattn-tile-f32.cu +++ b/ggml-cuda/fattn-tile-f32.cu @@ -79,7 +79,7 @@ static __global__ void flash_attn_tile_ext_f32( #pragma unroll for (int i0 = 0; i0 < D; i0 += 2*WARP_SIZE) { - float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i0/2 + threadIdx.x]; + float2 tmp = ic0 + j < ne01 ? Q_f2[j*(nb01/sizeof(float2)) + i0/2 + threadIdx.x] : make_float2(0.0f, 0.0f); Q_f[j][i0 + 0*WARP_SIZE + threadIdx.x] = tmp.x * scale; Q_f[j][i0 + 1*WARP_SIZE + threadIdx.x] = tmp.y * scale; } diff --git a/ggml-cuda/fattn-vec-f16.cu b/ggml-cuda/fattn-vec-f16.cu index 7352dcabf..808e8f362 100644 --- a/ggml-cuda/fattn-vec-f16.cu +++ b/ggml-cuda/fattn-vec-f16.cu @@ -94,7 +94,7 @@ static __global__ void flash_attn_vec_ext_f16( for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { const int i = i0 + threadIdx.x; - const float2 tmp = Q_f2[j*(nb01/sizeof(float2)) + i]; + const float2 tmp = ncols <= 2 || ic0 + j < ne01 ? Q_f2[j*(nb01/sizeof(float2)) + i] : make_float2(0.0f, 0.0f); Q_h2[j][i0/WARP_SIZE] = make_half2(scale, scale) * make_half2(tmp.x, tmp.y); } } @@ -212,7 +212,7 @@ static __global__ void flash_attn_vec_ext_f16( #pragma unroll for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) { - if (ic0 + j_VKQ >= ne01) { + if (ncols > 2 && ic0 + j_VKQ >= ne01) { break; } @@ -227,7 +227,7 @@ static __global__ void flash_attn_vec_ext_f16( dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; } - if (parallel_blocks != 1 && tid < ncols && ic0 + tid < ne01) { + if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); } #else diff --git a/ggml-cuda/fattn-vec-f32.cu b/ggml-cuda/fattn-vec-f32.cu index 11476a6c0..b4652301b 100644 --- a/ggml-cuda/fattn-vec-f32.cu +++ b/ggml-cuda/fattn-vec-f32.cu @@ -91,7 +91,7 @@ static __global__ void flash_attn_vec_ext_f32( for (int i0 = 0; i0 < D/2; i0 += WARP_SIZE) { const int i = i0 + threadIdx.x; - Q_h2[j][i0/WARP_SIZE] = Q_f2[j*(nb01/sizeof(float2)) + i]; + Q_h2[j][i0/WARP_SIZE] = ncols <= 2 || ic0 + j ? Q_f2[j*(nb01/sizeof(float2)) + i] : make_float2(0.0f, 0.0f); Q_h2[j][i0/WARP_SIZE].x *= scale; Q_h2[j][i0/WARP_SIZE].y *= scale; } @@ -200,7 +200,7 @@ static __global__ void flash_attn_vec_ext_f32( #pragma unroll for (int j_VKQ = 0; j_VKQ < ncols; ++j_VKQ) { - if (ic0 + j_VKQ >= ne01) { + if (ncols > 2 && ic0 + j_VKQ >= ne01) { break; } @@ -215,7 +215,7 @@ static __global__ void flash_attn_vec_ext_f32( dst[j_dst*D*gridDim.y + D*blockIdx.y + tid] = dst_val; } - if (parallel_blocks != 1 && tid < ncols && ic0 + tid < ne01) { + if (parallel_blocks != 1 && tid < ncols && (ncols <= 2 || ic0 + tid < ne01)) { dst_meta[(ic0 + tid)*gridDim.y*parallel_blocks + blockIdx.y*parallel_blocks + ip] = make_float2(kqmax[tid], kqsum[tid]); } } From fbf777d2b9c30e7569e3d1c149501c1e31d9b5b9 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 23 May 2024 09:43:24 +0300 Subject: [PATCH 112/137] main : minor (#7462) --- examples/main/main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 791dc61a7..09fa85fce 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -474,12 +474,12 @@ int main(int argc, char ** argv) { LOG_TEE("\n\n"); if (params.interactive) { - const char *control_message; + const char * control_message; if (params.multiline_input) { - control_message = " - To return control to LLaMa, end your input with '\\'.\n" + control_message = " - To return control to the AI, end your input with '\\'.\n" " - To return control without starting a new line, end your input with '/'.\n"; } else { - control_message = " - Press Return to return control to LLaMa.\n" + control_message = " - Press Return to return control to the AI.\n" " - To return control without starting a new line, end your input with '/'.\n" " - If you want to submit another line, end your input with '\\'.\n"; } From 1b1e27cb49158123ef4902aa41eb368c9e76e6a1 Mon Sep 17 00:00:00 2001 From: 0cc4m Date: Thu, 23 May 2024 08:59:59 +0200 Subject: [PATCH 113/137] Update vulkan rope implementation to support frequency factors (#7475) --- ggml-vulkan-shaders.hpp | 1235 ++++++++++++++++++----------------- ggml-vulkan.cpp | 259 +++++--- ggml_vk_generate_shaders.py | 7 +- 3 files changed, 825 insertions(+), 676 deletions(-) diff --git a/ggml-vulkan-shaders.hpp b/ggml-vulkan-shaders.hpp index 70c4043d3..e8cb5f52c 100644 --- a/ggml-vulkan-shaders.hpp +++ b/ggml-vulkan-shaders.hpp @@ -78882,342 +78882,15 @@ const uint64_t rope_f32_len = 3072; unsigned char rope_neox_f16_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x5f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x75,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, 0x01,0x00,0x00,0x00,0x11,0x00,0x02,0x00,0x51,0x11,0x00,0x00, 0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00,0x47,0x4c,0x53,0x4c, 0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30,0x00,0x00,0x00,0x00, 0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x0f,0x00,0x0b,0x00,0x05,0x00,0x00,0x00,0x04,0x00,0x00,0x00, 0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, 0x68,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0xcf,0x00,0x00,0x00,0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00, -0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x29,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2a,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x05,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x08,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x30,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x68,0x00,0x00,0x00, -0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x95,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x96,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x96,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x96,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x98,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x98,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9b,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x9c,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0x9c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xcc,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x48,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xcf,0x00,0x00,0x00,0x22,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xcf,0x00,0x00,0x00, -0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x16,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, -0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, -0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1b,0x00,0x00,0x00,0x6f,0x12,0x83,0x3a,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x00,0x00,0x80,0x3f, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, -0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, -0x1e,0x00,0x0c,0x00,0x2a,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x2a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2f,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x05,0x00,0x00,0x00,0x14,0x00,0x02,0x00,0x3c,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x41,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0xcd,0xcc,0xcc,0x3d,0x17,0x00,0x04,0x00,0x66,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x67,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x66,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x6a,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x73,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x16,0x00,0x03,0x00, -0x94,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x95,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x96,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x97,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x9b,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x9c,0x00,0x00,0x00, -0x9b,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x9c,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0x9d,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xa1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x94,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, -0xbc,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0xcc,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xce,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0xcd,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xce,0x00,0x00,0x00, -0xcf,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0xd1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0xda,0x00,0x00,0x00, -0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00, -0x66,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x69,0x00,0x00,0x00, -0x15,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x00,0x00,0x00,0x3f, -0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x17,0x01,0x00,0x00, -0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00,0x6f,0x00,0x00,0x00, -0x18,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x18,0x01,0x00,0x00, -0x41,0x00,0x05,0x00,0x6a,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x6c,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x6a,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x68,0x00,0x00,0x00, -0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x73,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00,0xae,0x00,0x05,0x00, -0x3c,0x00,0x00,0x00,0x76,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x76,0x00,0x00,0x00, -0x77,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, -0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x17,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x73,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x45,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x89,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0xac,0x00,0x05,0x00,0x3c,0x00,0x00,0x00,0x85,0x00,0x00,0x00, -0x7e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x85,0x00,0x00,0x00,0x86,0x00,0x00,0x00,0x87,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x86,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0x75,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0x90,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x8c,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00, -0xa2,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x94,0x00,0x00,0x00, -0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa1,0x00,0x00,0x00,0xa4,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xa4,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x69,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00, -0xa9,0x00,0x00,0x00,0x9e,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0xa6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x94,0x00,0x00,0x00, -0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa1,0x00,0x00,0x00,0xab,0x00,0x00,0x00,0x98,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0xa6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0xab,0x00,0x00,0x00,0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xb1,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0xb6,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0xbf,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, -0x83,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7f,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0xc8,0x00,0x00,0x00, -0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00,0xc9,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xc3,0x00,0x00,0x00, -0xc5,0x00,0x00,0x00,0x5b,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0xd1,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0xcf,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0xbf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0x6f,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, -0xd3,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x33,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0xda,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xde,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x5c,0x01,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0xdc,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0xe2,0x00,0x00,0x00,0xd9,0x00,0x00,0x00,0xe1,0x00,0x00,0x00, -0x6d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0xe5,0x00,0x00,0x00, -0xc9,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00, -0x22,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x2e,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x23,0x01,0x00,0x00, -0x22,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x26,0x01,0x00,0x00,0xd8,0x00,0x00,0x00,0xe2,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x2c,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x29,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0xb7,0x00,0x05,0x00,0x3c,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, -0x29,0x01,0x00,0x00,0x20,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x43,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x2a,0x01,0x00,0x00,0x2b,0x01,0x00,0x00,0x43,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x2b,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0x2f,0x00,0x00,0x00,0x2c,0x01,0x00,0x00,0x2c,0x00,0x00,0x00, -0x41,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x2d,0x01,0x00,0x00,0x2c,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x2f,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x2c,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x45,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x2f,0x01,0x00,0x00, -0x2e,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0x4f,0x01,0x00,0x00,0xe5,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x50,0x01,0x00,0x00, -0x4f,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x51,0x01,0x00,0x00,0x50,0x01,0x00,0x00,0x2d,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x2f,0x01,0x00,0x00,0x2d,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x52,0x01,0x00,0x00, -0x88,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x51,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x54,0x01,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x25,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x56,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x58,0x01,0x00,0x00,0x1f,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x33,0x01,0x00,0x00, -0x58,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x5d,0x01,0x00,0x00,0x57,0x01,0x00,0x00, -0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, -0x36,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x5d,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x39,0x01,0x00,0x00, -0xe2,0x00,0x00,0x00,0x33,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, -0x06,0x00,0x00,0x00,0x3a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x36,0x01,0x00,0x00, -0x39,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x3d,0x01,0x00,0x00,0x1f,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x53,0x00,0x00,0x00, -0x3e,0x01,0x00,0x00,0x1f,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x40,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x43,0x01,0x00,0x00, -0xf8,0x00,0x02,0x00,0x43,0x01,0x00,0x00,0xf5,0x00,0x07,0x00, -0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x23,0x01,0x00,0x00, -0x87,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x2b,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0x87,0x00,0x00,0x00,0x3a,0x01,0x00,0x00, -0x2b,0x01,0x00,0x00,0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x45,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x0e,0x00,0x00,0x00, -0x59,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x47,0x01,0x00,0x00,0x45,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, -0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x59,0x01,0x00,0x00, -0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x49,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0xa1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x94,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0xf1,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0xf7,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa1,0x00,0x00,0x00,0xf9,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x94,0x00,0x00,0x00,0xfa,0x00,0x00,0x00,0xf9,0x00,0x00,0x00, -0x73,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xfb,0x00,0x00,0x00, -0xfa,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x03,0x01,0x00,0x00,0xfb,0x00,0x00,0x00,0x4b,0x01,0x00,0x00, -0x7f,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x5e,0x01,0x00,0x00, -0x03,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, -0x04,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xf2,0x00,0x00,0x00,0x47,0x01,0x00,0x00,0x5e,0x01,0x00,0x00, -0x73,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x05,0x01,0x00,0x00, -0x04,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00, -0x06,0x01,0x00,0x00,0x98,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0xb9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0x06,0x01,0x00,0x00, -0x05,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x11,0x01,0x00,0x00,0xfb,0x00,0x00,0x00,0x47,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00,0x12,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xf2,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x11,0x01,0x00,0x00,0x73,0x00,0x04,0x00, -0x94,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0x12,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00,0x14,0x01,0x00,0x00, -0x98,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xf8,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x14,0x01,0x00,0x00,0x13,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x17,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x17,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, - -}; -const uint64_t rope_neox_f16_len = 3876; - -unsigned char rope_neox_f32_data[] = { -0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, -0x5a,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, -0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, -0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, -0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, -0x01,0x00,0x00,0x00,0x0f,0x00,0x0a,0x00,0x05,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x97,0x00,0x00,0x00, -0x9d,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x10,0x00,0x06,0x00, +0xcf,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x10,0x00,0x06,0x00, 0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x01,0x00,0x00,0x00, 0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, 0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, @@ -79237,51 +78910,61 @@ unsigned char rope_neox_f32_data[] = { 0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, 0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, 0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x09,0x00,0x00,0x00, -0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x47,0x00,0x03,0x00, -0x2a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x68,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x95,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x95,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x97,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x97,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x9b,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0x9b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x9b,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0x9d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, -0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, -0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xcc,0x00,0x00,0x00, -0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xce,0x00,0x00,0x00, -0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, -0xce,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, -0x47,0x00,0x04,0x00,0x11,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, -0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, -0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, -0x16,0x00,0x03,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x15,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x2a,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x2a,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x68,0x00,0x00,0x00, +0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x95,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x96,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x96,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x96,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x98,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x98,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x9b,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x9c,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0x9c,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9e,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xcc,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xcd,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xcd,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xcf,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xcf,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xdc,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x48,0x00,0x04,0x00,0xdd,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0xdd,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x03,0x00,0xdd,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xdf,0x00,0x00,0x00,0x22,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xdf,0x00,0x00,0x00, +0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x2b,0x01,0x00,0x00,0x0b,0x00,0x00,0x00,0x19,0x00,0x00,0x00, +0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00,0x21,0x00,0x03,0x00, +0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x16,0x00,0x03,0x00, +0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x15,0x00,0x04,0x00, +0x07,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x1b,0x00,0x00,0x00,0x6f,0x12,0x83,0x3a,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00,0x00,0x00,0x80,0x3f, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, 0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x6f,0x12,0x83,0x3a, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, -0x00,0x00,0x80,0x3f,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x04,0x00,0x00,0x00, -0x1c,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x28,0x00,0x00,0x00,0x1e,0x00,0x0c,0x00,0x2a,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x06,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1c,0x00,0x04,0x00, +0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x28,0x00,0x00,0x00, +0x1e,0x00,0x0d,0x00,0x2a,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x2b,0x00,0x00,0x00, 0x09,0x00,0x00,0x00,0x2a,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, 0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x09,0x00,0x00,0x00, 0x15,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x20,0x00,0x00,0x00, @@ -79306,228 +78989,614 @@ unsigned char rope_neox_f32_data[] = { 0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00, 0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x04,0x00, 0x73,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x07,0x00,0x00,0x00, -0x1d,0x00,0x03,0x00,0x94,0x00,0x00,0x00,0x06,0x00,0x00,0x00, -0x1e,0x00,0x03,0x00,0x95,0x00,0x00,0x00,0x94,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0x96,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x95,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x96,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, -0x9a,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, -0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00,0x20,0x00,0x04,0x00, -0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00, -0x3b,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa0,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x2d,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x09,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xcb,0x00,0x00,0x00, -0x2d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xcc,0x00,0x00,0x00, -0xcb,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xcd,0x00,0x00,0x00, -0x0c,0x00,0x00,0x00,0xcc,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, -0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, -0x20,0x00,0x04,0x00,0xd0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x94,0x00,0x00,0x00,0x10,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x95,0x00,0x00,0x00,0x94,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x96,0x00,0x00,0x00,0x95,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x96,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x97,0x00,0x00,0x00, +0x98,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0x9b,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0x9c,0x00,0x00,0x00,0x9b,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x9c,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x9d,0x00,0x00,0x00,0x9e,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xa1,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x94,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0xc1,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0xcc,0x00,0x00,0x00, +0x2d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0xcd,0x00,0x00,0x00, +0xcc,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xce,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0xcd,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0xce,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xd1,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, 0x2d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, -0xd9,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x00,0x01,0x00,0x00, -0x2c,0x00,0x06,0x00,0x66,0x00,0x00,0x00,0x11,0x01,0x00,0x00, -0x69,0x00,0x00,0x00,0x10,0x01,0x00,0x00,0x69,0x00,0x00,0x00, -0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, -0x00,0x00,0x00,0x3f,0x36,0x00,0x05,0x00,0x02,0x00,0x00,0x00, -0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x12,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0xfb,0x00,0x03,0x00, -0x6f,0x00,0x00,0x00,0x13,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x13,0x01,0x00,0x00,0x41,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x69,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, -0x6b,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x6a,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x68,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x70,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x74,0x00,0x00,0x00, -0xae,0x00,0x05,0x00,0x3c,0x00,0x00,0x00,0x76,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0xf7,0x00,0x03,0x00, -0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00, -0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00,0x78,0x00,0x00,0x00, -0xf8,0x00,0x02,0x00,0x77,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, -0x12,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x78,0x00,0x00,0x00, -0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x2c,0x00,0x00,0x00,0x45,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x07,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x7c,0x00,0x00,0x00, -0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x6d,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x89,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0xac,0x00,0x05,0x00,0x3c,0x00,0x00,0x00, -0x85,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x87,0x00,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x85,0x00,0x00,0x00,0x86,0x00,0x00,0x00, -0x87,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x86,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, -0x71,0x00,0x00,0x00,0x75,0x00,0x00,0x00,0x84,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x8c,0x00,0x00,0x00,0x90,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x91,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x93,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xa3,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x93,0x00,0x00,0x00,0x69,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa0,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0xa5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xa9,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, -0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, -0x97,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0xaa,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, -0xf9,0x00,0x02,0x00,0x12,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0xb0,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x75,0x00,0x00,0x00, -0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xb4,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xdc,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xdd,0x00,0x00,0x00,0xdc,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xde,0x00,0x00,0x00,0xdf,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xe2,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00,0x66,0x00,0x00,0x00, +0x2b,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0x2a,0x01,0x00,0x00, +0x69,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x72,0x01,0x00,0x00,0x00,0x00,0x00,0x3f,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x2c,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x6f,0x00,0x00,0x00,0x2d,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2d,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x6a,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x3c,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x2c,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0xac,0x00,0x05,0x00, +0x3c,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x87,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x85,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x86,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x90,0x00,0x00,0x00, 0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb0,0x00,0x00,0x00, -0xb4,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0xb7,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0xb5,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x73,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0xbb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x86,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x71,0x00,0x00,0x00, -0xbd,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xc0,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc2,0x00,0x00,0x00, -0xc1,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xc4,0x00,0x00,0x00,0x83,0x00,0x00,0x00,0x70,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, -0x7f,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0xc7,0x00,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, -0xc8,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xc2,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x56,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0xd0,0x00,0x00,0x00,0xd1,0x00,0x00,0x00, -0xce,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, -0xd1,0x00,0x00,0x00,0x6f,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd5,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x2f,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, -0x33,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xd5,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, -0xda,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0xdd,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0xdd,0x00,0x00,0x00, -0x57,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1a,0x00,0x00,0x00, -0xdb,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x85,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xd8,0x00,0x00,0x00, -0xe0,0x00,0x00,0x00,0x6d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, -0xe4,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x41,0x00,0x05,0x00, -0x2f,0x00,0x00,0x00,0x1d,0x01,0x00,0x00,0x2c,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00, +0xa4,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa4,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0x9e,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xa6,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00, +0xab,0x00,0x00,0x00,0x98,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0xa6,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xab,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0xb1,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0xb5,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xb6,0x00,0x00,0x00, +0xb1,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0xb9,0x00,0x00,0x00,0xb6,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0xbc,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0xbd,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xbf,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xbe,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2f,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0xc1,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc3,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc5,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7f,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0xc8,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x06,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xc3,0x00,0x00,0x00,0xc5,0x00,0x00,0x00, +0x71,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0xd1,0x00,0x00,0x00, +0xd2,0x00,0x00,0x00,0xcf,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0xbf,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, +0xd3,0x00,0x00,0x00,0xd2,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x73,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0xd5,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, +0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00,0xab,0x00,0x05,0x00, +0x3c,0x00,0x00,0x00,0xd8,0x00,0x00,0x00,0xd7,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xdb,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd8,0x00,0x00,0x00, +0xda,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xda,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xe2,0x00,0x00,0x00, +0xe3,0x00,0x00,0x00,0xdf,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe4,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xdb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe5,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xdb,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xdb,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0xe4,0x00,0x00,0x00,0xda,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0xe5,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xd3,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xec,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2f,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0xee,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xf1,0x00,0x00,0x00,0x72,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0xec,0x00,0x00,0x00,0xf4,0x00,0x00,0x00,0x88,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0xf5,0x00,0x00,0x00, +0x6e,0x01,0x00,0x00,0x6d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, +0xfa,0x00,0x00,0x00,0xc9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2f,0x00,0x00,0x00,0x37,0x01,0x00,0x00,0x2c,0x00,0x00,0x00, 0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x1e,0x01,0x00,0x00,0x1d,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0xd7,0x00,0x00,0x00, -0xe1,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00, -0x23,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x24,0x01,0x00,0x00, -0x23,0x01,0x00,0x00,0xb7,0x00,0x05,0x00,0x3c,0x00,0x00,0x00, -0x25,0x01,0x00,0x00,0x24,0x01,0x00,0x00,0x20,0x00,0x00,0x00, -0xf7,0x00,0x03,0x00,0x3e,0x01,0x00,0x00,0x00,0x00,0x00,0x00, -0xfa,0x00,0x04,0x00,0x25,0x01,0x00,0x00,0x26,0x01,0x00,0x00, -0x3e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x26,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0x2f,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x37,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x3b,0x01,0x00,0x00,0xeb,0x00,0x00,0x00, +0xf7,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3e,0x01,0x00,0x00, +0x3d,0x01,0x00,0x00,0xb7,0x00,0x05,0x00,0x3c,0x00,0x00,0x00, +0x3f,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x20,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x58,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x3f,0x01,0x00,0x00,0x40,0x01,0x00,0x00, +0x58,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x40,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x2f,0x00,0x00,0x00,0x41,0x01,0x00,0x00, 0x2c,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x42,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x28,0x01,0x00,0x00, -0x27,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x2f,0x00,0x00,0x00, -0x29,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x41,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x2f,0x00,0x00,0x00, +0x43,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x41,0x00,0x00,0x00, 0x45,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x2a,0x01,0x00,0x00,0x29,0x01,0x00,0x00,0x86,0x00,0x05,0x00, -0x07,0x00,0x00,0x00,0x4a,0x01,0x00,0x00,0xe4,0x00,0x00,0x00, +0x44,0x01,0x00,0x00,0x43,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0x64,0x01,0x00,0x00,0xfa,0x00,0x00,0x00, 0x17,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00, -0x4b,0x01,0x00,0x00,0x4a,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x4b,0x01,0x00,0x00, -0x28,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4d,0x01,0x00,0x00,0x2a,0x01,0x00,0x00,0x28,0x01,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x4e,0x01,0x00,0x00, +0x65,0x01,0x00,0x00,0x64,0x01,0x00,0x00,0x83,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x66,0x01,0x00,0x00,0x65,0x01,0x00,0x00, +0x42,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x67,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x42,0x01,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x68,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, -0x4d,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x4f,0x01,0x00,0x00,0x4c,0x01,0x00,0x00,0x4e,0x01,0x00,0x00, -0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x67,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x66,0x01,0x00,0x00,0x68,0x01,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x6b,0x01,0x00,0x00, 0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x20,0x00,0x00,0x00, -0x4f,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x25,0x00,0x00,0x00, -0x1f,0x00,0x00,0x00,0x51,0x01,0x00,0x00,0x83,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, -0x52,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x2e,0x01,0x00,0x00,0x53,0x01,0x00,0x00,0x24,0x01,0x00,0x00, -0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x58,0x01,0x00,0x00, -0x52,0x01,0x00,0x00,0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, -0x06,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x32,0x00,0x00,0x00,0x58,0x01,0x00,0x00,0x24,0x01,0x00,0x00, +0x69,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x6b,0x01,0x00,0x00,0x83,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0x6d,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x73,0x01,0x00,0x00, +0x6c,0x01,0x00,0x00,0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x06,0x00,0x00,0x00,0x4b,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x73,0x01,0x00,0x00,0x3e,0x01,0x00,0x00, 0x1f,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x34,0x01,0x00,0x00,0xe1,0x00,0x00,0x00,0x2e,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x21,0x01,0x00,0x00, -0x31,0x01,0x00,0x00,0x34,0x01,0x00,0x00,0x88,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, -0xd7,0x00,0x00,0x00,0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x39,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, -0x38,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, -0x3b,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0x53,0x00,0x00,0x00,0x39,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, -0x3e,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3e,0x01,0x00,0x00, -0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x55,0x01,0x00,0x00, -0x1e,0x01,0x00,0x00,0x87,0x00,0x00,0x00,0x3d,0x01,0x00,0x00, -0x26,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x21,0x01,0x00,0x00,0x87,0x00,0x00,0x00, -0x35,0x01,0x00,0x00,0x26,0x01,0x00,0x00,0x0c,0x00,0x06,0x00, -0x06,0x00,0x00,0x00,0x40,0x01,0x00,0x00,0x01,0x00,0x00,0x00, -0x0e,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x42,0x01,0x00,0x00,0x40,0x01,0x00,0x00, -0x55,0x01,0x00,0x00,0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00, -0x44,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, -0x54,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, -0x46,0x01,0x00,0x00,0x44,0x01,0x00,0x00,0x55,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00,0xef,0x00,0x00,0x00, +0x4e,0x01,0x00,0x00,0xf7,0x00,0x00,0x00,0x48,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00,0x4f,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x4b,0x01,0x00,0x00,0x4e,0x01,0x00,0x00,0x88,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x52,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x53,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x52,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, +0x55,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x53,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0x55,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x58,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x58,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x70,0x01,0x00,0x00, +0x38,0x01,0x00,0x00,0xdb,0x00,0x00,0x00,0x57,0x01,0x00,0x00, +0x40,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x3b,0x01,0x00,0x00,0xdb,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x40,0x01,0x00,0x00,0x0c,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x5a,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x6f,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x5c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00, +0x70,0x01,0x00,0x00,0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x5e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x6f,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x60,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x70,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00,0x05,0x01,0x00,0x00, +0x9e,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xb9,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x06,0x01,0x00,0x00, +0x05,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x06,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x7d,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0xb9,0x00,0x00,0x00,0x0c,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00,0x0e,0x01,0x00,0x00, +0x9e,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x0d,0x01,0x00,0x00, +0x3d,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, +0x0e,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x10,0x01,0x00,0x00,0x0f,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x18,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x60,0x01,0x00,0x00,0x7f,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x74,0x01,0x00,0x00,0x18,0x01,0x00,0x00,0x0c,0x00,0x08,0x00, +0x06,0x00,0x00,0x00,0x19,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x07,0x01,0x00,0x00,0x5c,0x01,0x00,0x00, +0x74,0x01,0x00,0x00,0x73,0x00,0x04,0x00,0x94,0x00,0x00,0x00, +0x1a,0x01,0x00,0x00,0x19,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xa1,0x00,0x00,0x00,0x1b,0x01,0x00,0x00,0x98,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0xb9,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, +0x1b,0x01,0x00,0x00,0x1a,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x10,0x01,0x00,0x00, +0x5c,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, +0x27,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x07,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x26,0x01,0x00,0x00, +0x73,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x28,0x01,0x00,0x00, +0x27,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0xa1,0x00,0x00,0x00, +0x29,0x01,0x00,0x00,0x98,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x0d,0x01,0x00,0x00,0x3e,0x00,0x03,0x00,0x29,0x01,0x00,0x00, +0x28,0x01,0x00,0x00,0xf9,0x00,0x02,0x00,0x2c,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x2c,0x01,0x00,0x00,0xfd,0x00,0x01,0x00, +0x38,0x00,0x01,0x00, +}; +const uint64_t rope_neox_f16_len = 4300; + +unsigned char rope_neox_f32_data[] = { +0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, +0x6f,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x02,0x00, +0x01,0x00,0x00,0x00,0x0b,0x00,0x06,0x00,0x01,0x00,0x00,0x00, +0x47,0x4c,0x53,0x4c,0x2e,0x73,0x74,0x64,0x2e,0x34,0x35,0x30, +0x00,0x00,0x00,0x00,0x0e,0x00,0x03,0x00,0x00,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x0f,0x00,0x0b,0x00,0x05,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x6d,0x61,0x69,0x6e,0x00,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x97,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0xde,0x00,0x00,0x00, +0x10,0x00,0x06,0x00,0x04,0x00,0x00,0x00,0x11,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x2a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x08,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x2a,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x14,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x18,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x1c,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x2a,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x23,0x00,0x00,0x00,0x30,0x00,0x00,0x00, +0x48,0x00,0x05,0x00,0x2a,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x23,0x00,0x00,0x00,0x34,0x00,0x00,0x00,0x47,0x00,0x03,0x00, +0x2a,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x68,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x94,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x95,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x19,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x95,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x97,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x97,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x03,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x9a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0x9b,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0x9b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0x9b,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0x9d,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0x9d,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xcb,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xcc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xcc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xcc,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xce,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xce,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0xdb,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x04,0x00,0x00,0x00,0x48,0x00,0x04,0x00,0xdc,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x48,0x00,0x05,0x00, +0xdc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x47,0x00,0x03,0x00,0xdc,0x00,0x00,0x00, +0x02,0x00,0x00,0x00,0x47,0x00,0x04,0x00,0xde,0x00,0x00,0x00, +0x22,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x47,0x00,0x04,0x00, +0xde,0x00,0x00,0x00,0x21,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x47,0x00,0x04,0x00,0x25,0x01,0x00,0x00,0x0b,0x00,0x00,0x00, +0x19,0x00,0x00,0x00,0x13,0x00,0x02,0x00,0x02,0x00,0x00,0x00, +0x21,0x00,0x03,0x00,0x03,0x00,0x00,0x00,0x02,0x00,0x00,0x00, +0x16,0x00,0x03,0x00,0x06,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x15,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x6f,0x12,0x83,0x3a, +0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x1f,0x00,0x00,0x00, +0x00,0x00,0x80,0x3f,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x07,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x04,0x00,0x00,0x00, +0x1c,0x00,0x04,0x00,0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x28,0x00,0x00,0x00,0x1e,0x00,0x0d,0x00,0x2a,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0x2b,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x2a,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0x2b,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0x09,0x00,0x00,0x00,0x15,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, +0x20,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0x2e,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x2f,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, +0x33,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0x39,0x00,0x00,0x00,0x05,0x00,0x00,0x00, +0x14,0x00,0x02,0x00,0x3c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x53,0x00,0x00,0x00,0xcd,0xcc,0xcc,0x3d, +0x17,0x00,0x04,0x00,0x66,0x00,0x00,0x00,0x07,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x67,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x66,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x67,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x6a,0x00,0x00,0x00, +0x01,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x07,0x00,0x00,0x00,0x6f,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x73,0x00,0x00,0x00,0x09,0x00,0x00,0x00, +0x07,0x00,0x00,0x00,0x1d,0x00,0x03,0x00,0x94,0x00,0x00,0x00, +0x06,0x00,0x00,0x00,0x1e,0x00,0x03,0x00,0x95,0x00,0x00,0x00, +0x94,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0x96,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x95,0x00,0x00,0x00,0x3b,0x00,0x04,0x00, +0x96,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0x9a,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0x9b,0x00,0x00,0x00,0x9a,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0x9c,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0x9b,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0x9c,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xa0,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00,0xbb,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x1d,0x00,0x03,0x00, +0xcb,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x1e,0x00,0x03,0x00, +0xcc,0x00,0x00,0x00,0xcb,0x00,0x00,0x00,0x20,0x00,0x04,0x00, +0xcd,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0xcc,0x00,0x00,0x00, +0x3b,0x00,0x04,0x00,0xcd,0x00,0x00,0x00,0xce,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x20,0x00,0x04,0x00,0xd0,0x00,0x00,0x00, +0x0c,0x00,0x00,0x00,0x2d,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x0a,0x00,0x00,0x00, +0x1d,0x00,0x03,0x00,0xdb,0x00,0x00,0x00,0x06,0x00,0x00,0x00, +0x1e,0x00,0x03,0x00,0xdc,0x00,0x00,0x00,0xdb,0x00,0x00,0x00, +0x20,0x00,0x04,0x00,0xdd,0x00,0x00,0x00,0x0c,0x00,0x00,0x00, +0xdc,0x00,0x00,0x00,0x3b,0x00,0x04,0x00,0xdd,0x00,0x00,0x00, +0xde,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x2b,0x00,0x04,0x00, +0x2d,0x00,0x00,0x00,0xeb,0x00,0x00,0x00,0x08,0x00,0x00,0x00, +0x2b,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x00,0x01,0x00,0x00,0x2c,0x00,0x06,0x00,0x66,0x00,0x00,0x00, +0x25,0x01,0x00,0x00,0x69,0x00,0x00,0x00,0x24,0x01,0x00,0x00, +0x69,0x00,0x00,0x00,0x2b,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6c,0x01,0x00,0x00,0x00,0x00,0x00,0x3f,0x36,0x00,0x05,0x00, +0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0x03,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x05,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x26,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfb,0x00,0x03,0x00,0x6f,0x00,0x00,0x00,0x27,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x27,0x01,0x00,0x00,0x41,0x00,0x05,0x00, +0x6a,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x68,0x00,0x00,0x00, +0x69,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, +0x6c,0x00,0x00,0x00,0x6b,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x6c,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x6a,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x68,0x00,0x00,0x00,0x6f,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x70,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x74,0x00,0x00,0x00,0xae,0x00,0x05,0x00,0x3c,0x00,0x00,0x00, +0x76,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x76,0x00,0x00,0x00,0x77,0x00,0x00,0x00, +0x78,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0x77,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0x26,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x78,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x45,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x7c,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x6d,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x89,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x6d,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0xac,0x00,0x05,0x00, +0x3c,0x00,0x00,0x00,0x85,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0x87,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0x85,0x00,0x00,0x00, +0x86,0x00,0x00,0x00,0x87,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0x86,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0x8c,0x00,0x00,0x00,0x71,0x00,0x00,0x00,0x75,0x00,0x00,0x00, +0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x90,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x80,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x8c,0x00,0x00,0x00, +0x90,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x91,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00,0xa1,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x93,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa2,0x00,0x00,0x00, +0xa1,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00, +0xa3,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x93,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xa3,0x00,0x00,0x00, +0xa2,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x93,0x00,0x00,0x00,0x69,0x00,0x00,0x00, +0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00,0xa8,0x00,0x00,0x00, +0x9d,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xa5,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xa9,0x00,0x00,0x00, +0xa8,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00, +0xaa,0x00,0x00,0x00,0x97,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0xa5,0x00,0x00,0x00,0x3e,0x00,0x03,0x00,0xaa,0x00,0x00,0x00, +0xa9,0x00,0x00,0x00,0xf9,0x00,0x02,0x00,0x26,0x01,0x00,0x00, +0xf8,0x00,0x02,0x00,0x87,0x00,0x00,0x00,0x84,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x71,0x00,0x00,0x00, +0x75,0x00,0x00,0x00,0x84,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0xb4,0x00,0x00,0x00,0x7e,0x00,0x00,0x00,0x7d,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xb5,0x00,0x00,0x00, +0xb0,0x00,0x00,0x00,0xb4,0x00,0x00,0x00,0x86,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0xb7,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0xb8,0x00,0x00,0x00,0xb5,0x00,0x00,0x00,0xb7,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x73,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0xbb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x07,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0xbc,0x00,0x00,0x00, +0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xbe,0x00,0x00,0x00, +0x71,0x00,0x00,0x00,0xbd,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2f,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0xc0,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xc2,0x00,0x00,0x00,0xc1,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x83,0x00,0x00,0x00, +0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xc7,0x00,0x00,0x00, +0x7e,0x00,0x00,0x00,0x7f,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0xc7,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x06,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0xc2,0x00,0x00,0x00,0xc4,0x00,0x00,0x00, +0x6b,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0xd0,0x00,0x00,0x00, +0xd1,0x00,0x00,0x00,0xce,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0xbe,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x2d,0x00,0x00,0x00, +0xd2,0x00,0x00,0x00,0xd1,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x73,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0xd4,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, +0xd6,0x00,0x00,0x00,0xd5,0x00,0x00,0x00,0xab,0x00,0x05,0x00, +0x3c,0x00,0x00,0x00,0xd7,0x00,0x00,0x00,0xd6,0x00,0x00,0x00, +0x6f,0x00,0x00,0x00,0xf7,0x00,0x03,0x00,0xda,0x00,0x00,0x00, +0x00,0x00,0x00,0x00,0xfa,0x00,0x04,0x00,0xd7,0x00,0x00,0x00, +0xd9,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xd9,0x00,0x00,0x00,0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00, +0xe1,0x00,0x00,0x00,0xde,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0xb7,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xe2,0x00,0x00,0x00,0xe1,0x00,0x00,0x00,0xf9,0x00,0x02,0x00, +0xda,0x00,0x00,0x00,0xf8,0x00,0x02,0x00,0xe3,0x00,0x00,0x00, +0xf9,0x00,0x02,0x00,0xda,0x00,0x00,0x00,0xf8,0x00,0x02,0x00, +0xda,0x00,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0xe2,0x00,0x00,0x00,0xd9,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0xe3,0x00,0x00,0x00,0x6f,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe7,0x00,0x00,0x00,0xd2,0x00,0x00,0x00, +0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x2c,0x00,0x00,0x00,0x33,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0xe8,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xea,0x00,0x00,0x00, +0xe7,0x00,0x00,0x00,0xe9,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2f,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x2c,0x00,0x00,0x00, +0xeb,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0xed,0x00,0x00,0x00,0xec,0x00,0x00,0x00,0x70,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0xef,0x00,0x00,0x00,0x6d,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0xef,0x00,0x00,0x00,0x6c,0x01,0x00,0x00,0x0c,0x00,0x07,0x00, +0x06,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x01,0x00,0x00,0x00, +0x1a,0x00,0x00,0x00,0xed,0x00,0x00,0x00,0xf1,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0xea,0x00,0x00,0x00,0xf2,0x00,0x00,0x00,0x88,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0xf3,0x00,0x00,0x00, +0x68,0x01,0x00,0x00,0x6d,0x00,0x04,0x00,0x07,0x00,0x00,0x00, +0xf8,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x41,0x00,0x05,0x00, +0x2f,0x00,0x00,0x00,0x31,0x01,0x00,0x00,0x2c,0x00,0x00,0x00, +0x2e,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x32,0x01,0x00,0x00,0x31,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x35,0x01,0x00,0x00,0xe9,0x00,0x00,0x00, +0xf5,0x00,0x00,0x00,0x41,0x00,0x05,0x00,0x2f,0x00,0x00,0x00, +0x37,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x39,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x38,0x01,0x00,0x00, +0x37,0x01,0x00,0x00,0xb7,0x00,0x05,0x00,0x3c,0x00,0x00,0x00, +0x39,0x01,0x00,0x00,0x38,0x01,0x00,0x00,0x20,0x00,0x00,0x00, +0xf7,0x00,0x03,0x00,0x52,0x01,0x00,0x00,0x00,0x00,0x00,0x00, +0xfa,0x00,0x04,0x00,0x39,0x01,0x00,0x00,0x3a,0x01,0x00,0x00, +0x52,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x3a,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0x2f,0x00,0x00,0x00,0x3b,0x01,0x00,0x00, +0x2c,0x00,0x00,0x00,0x41,0x00,0x00,0x00,0x42,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x3c,0x01,0x00,0x00, +0x3b,0x01,0x00,0x00,0x41,0x00,0x06,0x00,0x2f,0x00,0x00,0x00, +0x3d,0x01,0x00,0x00,0x2c,0x00,0x00,0x00,0x41,0x00,0x00,0x00, +0x45,0x00,0x00,0x00,0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x3e,0x01,0x00,0x00,0x3d,0x01,0x00,0x00,0x86,0x00,0x05,0x00, +0x07,0x00,0x00,0x00,0x5e,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, +0x17,0x00,0x00,0x00,0x70,0x00,0x04,0x00,0x06,0x00,0x00,0x00, +0x5f,0x01,0x00,0x00,0x5e,0x01,0x00,0x00,0x83,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x60,0x01,0x00,0x00,0x5f,0x01,0x00,0x00, +0x3c,0x01,0x00,0x00,0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x3e,0x01,0x00,0x00,0x3c,0x01,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x62,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x1b,0x00,0x00,0x00, +0x61,0x01,0x00,0x00,0x88,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x60,0x01,0x00,0x00,0x62,0x01,0x00,0x00, +0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x65,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x28,0x00,0x00,0x00,0x20,0x00,0x00,0x00, +0x63,0x01,0x00,0x00,0x0c,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x66,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x25,0x00,0x00,0x00, +0x1f,0x00,0x00,0x00,0x65,0x01,0x00,0x00,0x83,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x67,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, +0x66,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x42,0x01,0x00,0x00,0x67,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x83,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x6d,0x01,0x00,0x00, +0x66,0x01,0x00,0x00,0x1f,0x00,0x00,0x00,0x0c,0x00,0x08,0x00, +0x06,0x00,0x00,0x00,0x45,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x32,0x00,0x00,0x00,0x6d,0x01,0x00,0x00,0x38,0x01,0x00,0x00, +0x1f,0x00,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x48,0x01,0x00,0x00,0xf5,0x00,0x00,0x00,0x42,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00,0x49,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x35,0x01,0x00,0x00, +0x45,0x01,0x00,0x00,0x48,0x01,0x00,0x00,0x88,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x4c,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, +0xe9,0x00,0x00,0x00,0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x4d,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x1c,0x00,0x00,0x00, +0x4c,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, +0x4f,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x53,0x00,0x00,0x00,0x4d,0x01,0x00,0x00,0x1f,0x00,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0x4f,0x01,0x00,0x00,0xf9,0x00,0x02,0x00, +0x52,0x01,0x00,0x00,0xf8,0x00,0x02,0x00,0x52,0x01,0x00,0x00, +0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00,0x6a,0x01,0x00,0x00, +0x32,0x01,0x00,0x00,0xda,0x00,0x00,0x00,0x51,0x01,0x00,0x00, +0x3a,0x01,0x00,0x00,0xf5,0x00,0x07,0x00,0x06,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x35,0x01,0x00,0x00,0xda,0x00,0x00,0x00, +0x49,0x01,0x00,0x00,0x3a,0x01,0x00,0x00,0x0c,0x00,0x06,0x00, +0x06,0x00,0x00,0x00,0x54,0x01,0x00,0x00,0x01,0x00,0x00,0x00, +0x0e,0x00,0x00,0x00,0x69,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x56,0x01,0x00,0x00,0x54,0x01,0x00,0x00, +0x6a,0x01,0x00,0x00,0x0c,0x00,0x06,0x00,0x06,0x00,0x00,0x00, +0x58,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x0d,0x00,0x00,0x00, +0x69,0x01,0x00,0x00,0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00, +0x5a,0x01,0x00,0x00,0x58,0x01,0x00,0x00,0x6a,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00,0x03,0x01,0x00,0x00, 0x9d,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xb8,0x00,0x00,0x00, -0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0xef,0x00,0x00,0x00,0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00, -0xf5,0x00,0x00,0x00,0x7d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, -0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0xb8,0x00,0x00,0x00,0xf5,0x00,0x00,0x00,0x41,0x00,0x06,0x00, -0xa0,0x00,0x00,0x00,0xf7,0x00,0x00,0x00,0x9d,0x00,0x00,0x00, -0x42,0x00,0x00,0x00,0xf6,0x00,0x00,0x00,0x3d,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0xf7,0x00,0x00,0x00, -0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x00,0x01,0x00,0x00, -0xf8,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x7f,0x00,0x04,0x00, -0x06,0x00,0x00,0x00,0x59,0x01,0x00,0x00,0x00,0x01,0x00,0x00, -0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00,0x01,0x01,0x00,0x00, -0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0xf0,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x59,0x01,0x00,0x00,0x41,0x00,0x06,0x00, -0xa0,0x00,0x00,0x00,0x02,0x01,0x00,0x00,0x97,0x00,0x00,0x00, +0x3d,0x00,0x04,0x00,0x06,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x03,0x01,0x00,0x00,0x86,0x00,0x05,0x00,0x07,0x00,0x00,0x00, +0x09,0x01,0x00,0x00,0x7d,0x00,0x00,0x00,0x17,0x00,0x00,0x00, +0x80,0x00,0x05,0x00,0x07,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0xb8,0x00,0x00,0x00,0x09,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xa0,0x00,0x00,0x00,0x0b,0x01,0x00,0x00,0x9d,0x00,0x00,0x00, +0x42,0x00,0x00,0x00,0x0a,0x01,0x00,0x00,0x3d,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x0c,0x01,0x00,0x00,0x0b,0x01,0x00,0x00, +0x85,0x00,0x05,0x00,0x06,0x00,0x00,0x00,0x14,0x01,0x00,0x00, +0x0c,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x7f,0x00,0x04,0x00, +0x06,0x00,0x00,0x00,0x6e,0x01,0x00,0x00,0x14,0x01,0x00,0x00, +0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00,0x15,0x01,0x00,0x00, +0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x04,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x6e,0x01,0x00,0x00,0x41,0x00,0x06,0x00, +0xa0,0x00,0x00,0x00,0x16,0x01,0x00,0x00,0x97,0x00,0x00,0x00, 0x42,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x3e,0x00,0x03,0x00, -0x02,0x01,0x00,0x00,0x01,0x01,0x00,0x00,0x85,0x00,0x05,0x00, -0x06,0x00,0x00,0x00,0x0d,0x01,0x00,0x00,0xf8,0x00,0x00,0x00, -0x42,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, -0x0e,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, -0xf0,0x00,0x00,0x00,0x46,0x01,0x00,0x00,0x0d,0x01,0x00,0x00, -0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00,0x0f,0x01,0x00,0x00, -0x97,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0xf6,0x00,0x00,0x00, -0x3e,0x00,0x03,0x00,0x0f,0x01,0x00,0x00,0x0e,0x01,0x00,0x00, -0xf9,0x00,0x02,0x00,0x12,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, -0x12,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, +0x16,0x01,0x00,0x00,0x15,0x01,0x00,0x00,0x85,0x00,0x05,0x00, +0x06,0x00,0x00,0x00,0x21,0x01,0x00,0x00,0x0c,0x01,0x00,0x00, +0x56,0x01,0x00,0x00,0x0c,0x00,0x08,0x00,0x06,0x00,0x00,0x00, +0x22,0x01,0x00,0x00,0x01,0x00,0x00,0x00,0x32,0x00,0x00,0x00, +0x04,0x01,0x00,0x00,0x5a,0x01,0x00,0x00,0x21,0x01,0x00,0x00, +0x41,0x00,0x06,0x00,0xa0,0x00,0x00,0x00,0x23,0x01,0x00,0x00, +0x97,0x00,0x00,0x00,0x42,0x00,0x00,0x00,0x0a,0x01,0x00,0x00, +0x3e,0x00,0x03,0x00,0x23,0x01,0x00,0x00,0x22,0x01,0x00,0x00, +0xf9,0x00,0x02,0x00,0x26,0x01,0x00,0x00,0xf8,0x00,0x02,0x00, +0x26,0x01,0x00,0x00,0xfd,0x00,0x01,0x00,0x38,0x00,0x01,0x00, }; -const uint64_t rope_neox_f32_len = 3792; +const uint64_t rope_neox_f32_len = 4200; unsigned char scale_f32_data[] = { 0x03,0x02,0x23,0x07,0x00,0x05,0x01,0x00,0x0b,0x00,0x0d,0x00, diff --git a/ggml-vulkan.cpp b/ggml-vulkan.cpp index 16287a280..79ce1479f 100644 --- a/ggml-vulkan.cpp +++ b/ggml-vulkan.cpp @@ -290,6 +290,7 @@ struct vk_op_rope_neox_push_constants { float corr_dims[4]; float theta_scale; float inv_ndims; + uint32_t has_freq_facs; }; struct vk_op_soft_max_push_constants { @@ -1522,8 +1523,8 @@ static void ggml_vk_load_shaders(ggml_backend_vk_context * ctx) { ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_f32, "rope_f32", rope_f32_len, rope_f32_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_f16, "rope_f16", rope_f16_len, rope_f16_data, "main", 3, sizeof(vk_op_rope_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_neox_f32, "rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); - ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_neox_f16, "rope_neox_f16", rope_neox_f16_len, rope_neox_f16_data, "main", 3, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_neox_f32, "rope_neox_f32", rope_neox_f32_len, rope_neox_f32_data, "main", 4, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); + ggml_vk_create_pipeline(ctx, ctx->device->pipeline_rope_neox_f16, "rope_neox_f16", rope_neox_f16_len, rope_neox_f16_data, "main", 4, sizeof(vk_op_rope_neox_push_constants), {1, 512, 1}, {}, 1); ggml_vk_create_pipeline(ctx, ctx->device->pipeline_argsort_f32, "argsort_f32", argsort_f32_len, argsort_f32_data, "main", 2, sizeof(vk_op_argsort_push_constants), {1024, 1, 1}, {}, 1); } @@ -3732,7 +3733,7 @@ static void ggml_vk_op_repeat(ggml_backend_vk_context * ctx, vk_context * subctx } -static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op) { +static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, ggml_op op) { switch (op) { case GGML_OP_ADD: if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { @@ -3853,6 +3854,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const default: return nullptr; } + + GGML_UNUSED(src2); } static ggml_vk_func_t ggml_vk_op_get_func(ggml_op op) { @@ -3880,12 +3883,15 @@ static bool ggml_vk_op_supports_incontiguous(ggml_op op) { } template -static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, ggml_op op, const PC&& pc) { +static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst, ggml_op op, const PC&& pc) { #ifdef GGML_VULKAN_DEBUG std::cerr << "ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3]; if (src1 != nullptr) { std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3]; } + if (src2 != nullptr) { + std::cerr << "), (" << src2 << ", name=" << src2->name << ", type=" << src2->type << ", ne0=" << src2->ne[0] << ", ne1=" << src2->ne[1] << ", ne2=" << src2->ne[2] << ", ne3=" << src2->ne[3] << ", nb0=" << src2->nb[0] << ", nb1=" << src2->nb[1] << ", nb2=" << src2->nb[2] << ", nb3=" << src2->nb[3]; + } std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3] << "), " << ggml_op_name(op) << ")" << std::endl; #endif GGML_ASSERT(op == GGML_OP_GET_ROWS || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT @@ -3896,6 +3902,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c const uint64_t ne02 = src0->ne[2]; const uint64_t ne03 = src0->ne[3]; const uint64_t ne0 = ne00 * ne01; + const bool use_src1 = src1 != nullptr; const uint64_t ne10 = use_src1 ? src1->ne[0] : 0; const uint64_t ne11 = use_src1 ? src1->ne[1] : 0; @@ -3904,7 +3911,14 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c const uint64_t ne1 = ne10 * ne11; // const uint64_t nb10 = use_src1 ? src1->nb[0] : 0; - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, dst, op); + const bool use_src2 = src2 != nullptr; + const uint64_t ne20 = use_src2 ? src2->ne[0] : 0; + const uint64_t ne21 = use_src2 ? src2->ne[1] : 0; + const uint64_t ne22 = use_src2 ? src2->ne[2] : 0; + const uint64_t ne23 = use_src2 ? src2->ne[3] : 0; + const uint64_t ne2 = ne20 * ne21; + + vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op); ggml_vk_func_t op_func; if (pipeline == nullptr) { @@ -3927,15 +3941,18 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) dst->extra; ggml_tensor_extra_gpu * extra_src0 = (ggml_tensor_extra_gpu *) src0->extra; ggml_tensor_extra_gpu * extra_src1 = use_src1 ? (ggml_tensor_extra_gpu *) src1->extra : nullptr; + ggml_tensor_extra_gpu * extra_src2 = use_src2 ? (ggml_tensor_extra_gpu *) src2->extra : nullptr; vk_buffer d_X = nullptr; size_t x_buf_offset = 0; vk_buffer d_Y = nullptr; size_t y_buf_offset = 0; vk_buffer d_Z = nullptr; + size_t z_buf_offset = 0; bool src0_uma = false; bool src1_uma = false; + bool src2_uma = false; if (ctx->device->uma) { ggml_vk_host_get(ctx, src0->data, d_X, x_buf_offset); @@ -3944,10 +3961,15 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_host_get(ctx, src1->data, d_Y, y_buf_offset); src1_uma = d_Y != nullptr; } + if (use_src2) { + ggml_vk_host_get(ctx, src2->data, d_Z, z_buf_offset); + src2_uma = d_Z != nullptr; + } } uint64_t x_sz = ggml_vk_align_size(ggml_type_size(src0->type)/ggml_blck_size(src0->type) * ne0, ctx->device->properties.limits.minStorageBufferOffsetAlignment); uint64_t y_sz = use_src1 ? ggml_vk_align_size(ggml_type_size(src1->type) * ne1, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : 0; + uint64_t z_sz = use_src2 ? ggml_vk_align_size(ggml_type_size(src2->type) * ne2, ctx->device->properties.limits.minStorageBufferOffsetAlignment) : 0; uint64_t d_sz = ggml_type_size(dst->type) * ne0; vk_buffer d_D = extra->buffer_gpu.lock(); @@ -3970,10 +3992,16 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c y_buf_offset = extra_src1->offset; GGML_ASSERT(d_Y != nullptr); } + if (use_src2 && !src2_uma) { + d_Z = extra_src2->buffer_gpu.lock(); + z_buf_offset = extra_src2->offset; + GGML_ASSERT(d_Z != nullptr); + } if (op_supports_incontiguous) { x_sz = ggml_nbytes(src0); y_sz = use_src1 ? ggml_nbytes(src1) : 0; + z_sz = use_src2 ? ggml_nbytes(src2) : 0; d_sz = ggml_nbytes(dst); if (x_buf_offset + x_sz >= d_X->size) { @@ -3982,6 +4010,9 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c if (use_src1 && y_buf_offset + y_sz >= d_Y->size) { y_sz = VK_WHOLE_SIZE; } + if (use_src2 && z_buf_offset + z_sz >= d_Z->size) { + z_sz = VK_WHOLE_SIZE; + } if (d_buf_offset + d_sz >= d_D->size) { d_sz = VK_WHOLE_SIZE; } @@ -4021,13 +4052,16 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c if (use_src1 && y_sz != VK_WHOLE_SIZE) { y_sz *= ne12 * ne13; } + if (use_src2 && z_sz != VK_WHOLE_SIZE) { + z_sz *= ne22 * ne23; + } if (d_sz != VK_WHOLE_SIZE) { d_sz *= ne02 * ne03; } } if (op == GGML_OP_SOFT_MAX) { - // Empty src1 is possible on soft_max, but the shader needs a buffer + // Empty src1 is possible in soft_max, but the shader needs a buffer vk_subbuffer subbuf_y; if (use_src1) { subbuf_y = { d_Y, y_buf_offset, y_sz }; @@ -4037,6 +4071,28 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, subbuf_y, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + } else if (op == GGML_OP_ROPE) { + const int mode = ((int32_t *) dst->op_params)[2]; + const bool is_neox = mode & 2; + + if (is_neox) { + // Empty src2 is possible in rope, but the shader needs a buffer + vk_subbuffer subbuf_z; + if (use_src2) { + subbuf_z = { d_Z, z_buf_offset, z_sz }; + } else { + subbuf_z = { d_X, 0, d_X->size }; + } + + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, subbuf_z, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + } else { + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); + } + } else if (use_src2) { + ggml_vk_sync_buffers(subctx); + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, { d_Z, z_buf_offset, z_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); } else if (use_src1) { ggml_vk_sync_buffers(subctx); ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { { d_X, x_buf_offset, x_sz }, { d_Y, y_buf_offset, y_sz }, { d_D, d_buf_offset, d_sz } }, sizeof(PC), &pc, elements); @@ -4047,6 +4103,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } else { GGML_ASSERT(op != GGML_OP_SOFT_MAX); GGML_ASSERT(op != GGML_OP_ARGSORT); + GGML_ASSERT(!use_src2); ggml_pipeline_allocate_descriptor_sets(ctx, pipeline, ne02 * ne03); @@ -4088,7 +4145,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context * subctx, c } static void ggml_vk_repeat(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_REPEAT, { (uint32_t)ggml_nelements(src0), (uint32_t)ggml_nelements(src1), 0.0f, 0.0f }); } static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -4096,7 +4153,7 @@ static void ggml_vk_get_rows(ggml_backend_vk_context * ctx, vk_context * subctx, const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_GET_ROWS, { + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_GET_ROWS, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, @@ -4111,7 +4168,7 @@ static void ggml_vk_add(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ADD, { + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_ADD, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, @@ -4126,7 +4183,7 @@ static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src1_type_size = ggml_type_size(src1->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_MUL, { + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_MUL, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, @@ -4141,7 +4198,7 @@ static void ggml_vk_scale(ggml_backend_vk_context * ctx, vk_context * subctx, co const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SCALE, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_SCALE, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -4154,7 +4211,7 @@ static void ggml_vk_sqr(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_SQR, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_SQR, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -4168,7 +4225,7 @@ static void ggml_vk_clamp(ggml_backend_vk_context * ctx, vk_context * subctx, co const uint32_t src0_type_size = ggml_type_size(src0->type); const uint32_t dst_type_size = ggml_type_size(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CLAMP, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_CLAMP, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -4183,7 +4240,7 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context * subctx, cons const uint32_t dst_type_size = ggml_type_size(dst->type); const uint32_t d_offset = (extra->offset % ctx->device->properties.limits.minStorageBufferOffsetAlignment) / dst_type_size; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_CPY, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_CPY, { (uint32_t)ggml_nelements(src0), (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2], (uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, @@ -4195,21 +4252,21 @@ static void ggml_vk_cpy(ggml_backend_vk_context * ctx, vk_context * subctx, cons static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); } static void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { float * op_params = (float *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_RMS_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f }); } static void ggml_vk_unary(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_UNARY, { (uint32_t)ggml_nelements(src0), 0, 0.0f, 0.0f }); } static void ggml_vk_diag_mask_inf(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, ggml_tensor * dst) { int32_t * op_params = (int32_t *)dst->op_params; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_DIAG_MASK_INF, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0] }); } static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -4228,7 +4285,7 @@ static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, const float m0 = powf(2.0f, -(max_bias ) / n_head_log2); const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_SOFT_MAX, { + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, dst, GGML_OP_SOFT_MAX, { ncols, src1 != nullptr ? nrows_y : (uint32_t)0, scale, max_bias, @@ -4237,11 +4294,7 @@ static void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context * subctx, }); } -static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { -#pragma message("TODO: implement phi3 frequency factors support") -#pragma message(" https://github.com/ggerganov/llama.cpp/pull/7225") - GGML_ASSERT(dst->src[2] == nullptr && "phi3 frequency factors not implemented yet"); - +static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, ggml_tensor * dst) { const int n_dims = ((int32_t *) dst->op_params)[1]; const int mode = ((int32_t *) dst->op_params)[2]; // const int n_ctx = ((int32_t *) dst->op_params)[3]; @@ -4264,12 +4317,13 @@ static void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context * subctx, con if (is_neox) { const float theta_scale = powf(freq_base, -2.0f/n_dims); const float inv_ndims = -1.0f / n_dims; - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { + ggml_vk_op_f32(ctx, subctx, src0, src1, src2, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], (uint32_t)n_dims, freq_scale, (uint32_t)src0->ne[1], - freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1], 0.0f, 0.0f}, theta_scale, inv_ndims + freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1], 0.0f, 0.0f}, theta_scale, inv_ndims, + src2 != nullptr, }); } else { - ggml_vk_op_f32(ctx, subctx, src0, src1, dst, GGML_OP_ROPE, { + ggml_vk_op_f32(ctx, subctx, src0, src1, src2, dst, GGML_OP_ROPE, { (uint32_t)src0->ne[0], freq_scale, (uint32_t)src0->ne[1], freq_base, ext_factor, attn_factor, {corr_dims[0], corr_dims[1], 0.0f, 0.0f} }); @@ -4292,7 +4346,7 @@ static void ggml_vk_argsort(ggml_backend_vk_context * ctx, vk_context * subctx, std::cerr << ((ggml_sort_order) op_params[0]) << " " << GGML_SORT_ORDER_ASC << std::endl; - ggml_vk_op_f32(ctx, subctx, src0, nullptr, dst, GGML_OP_ARGSORT, { + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, dst, GGML_OP_ARGSORT, { ncols, ncols_pad, op_params[0], @@ -5408,6 +5462,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod const ggml_tensor * src0 = node->src[0]; const ggml_tensor * src1 = node->src[1]; + const ggml_tensor * src2 = node->src[2]; switch (node->op) { case GGML_OP_UNARY: @@ -5524,7 +5579,7 @@ static void ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_tensor * nod break; case GGML_OP_ROPE: - ggml_vk_rope(ctx, ctx->compute_ctx, src0, src1, node); + ggml_vk_rope(ctx, ctx->compute_ctx, src0, src1, src2, node); break; case GGML_OP_ARGSORT: @@ -6500,7 +6555,7 @@ static void ggml_vk_print_graph_origin(const ggml_tensor * tensor, std::vectorop) << " gpu=" << (tensor->extra != nullptr) << " backend=" << tensor->backend << std::endl; + std::cerr << ggml_op_name(tensor->op) << " gpu=" << (tensor->extra != nullptr) << std::endl; done.push_back(tensor); @@ -6550,7 +6605,7 @@ static void ggml_vk_print_tensor_area(const ggml_tensor * tensor, const void * d static void ggml_vk_print_tensor(ggml_backend_vk_context * ctx, const ggml_tensor * tensor, const char * name) { void * tensor_data = tensor->data; - if (tensor->backend == GGML_BACKEND_TYPE_GPU) { + if (ggml_backend_buffer_is_vk(tensor->buffer)) { const size_t tensor_size = ggml_nbytes(tensor); tensor_data = malloc(tensor_size); @@ -6561,12 +6616,12 @@ static void ggml_vk_print_tensor(ggml_backend_vk_context * ctx, const ggml_tenso } std::cerr << "TENSOR CHECK " << name << " (" << tensor->name << "): " << ggml_op_name(tensor->op) << std::endl; - std::cerr << "tensor=" << tensor << " tensor->backend: " << tensor->backend << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl; + std::cerr << "tensor=" << tensor << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl; if (tensor->src[0] != nullptr) { - std::cerr << "tensor->src[0]=" << tensor->src[0] << " name=" << tensor->src[0]->name << " op=" << ggml_op_name(tensor->src[0]->op) << " type=" << ggml_type_name(tensor->src[0]->type) << " backend=" << tensor->src[0]->backend << " ne0=" << tensor->src[0]->ne[0] << " nb0=" << tensor->src[0]->nb[0] << " ne1=" << tensor->src[0]->ne[1] << " nb1=" << tensor->src[0]->nb[1] << " ne2=" << tensor->src[0]->ne[2] << " nb2=" << tensor->src[0]->nb[2] << " ne3=" << tensor->src[0]->ne[3] << " nb3=" << tensor->src[0]->nb[3] << std::endl; + std::cerr << "tensor->src[0]=" << tensor->src[0] << " name=" << tensor->src[0]->name << " op=" << ggml_op_name(tensor->src[0]->op) << " type=" << ggml_type_name(tensor->src[0]->type) << " ne0=" << tensor->src[0]->ne[0] << " nb0=" << tensor->src[0]->nb[0] << " ne1=" << tensor->src[0]->ne[1] << " nb1=" << tensor->src[0]->nb[1] << " ne2=" << tensor->src[0]->ne[2] << " nb2=" << tensor->src[0]->nb[2] << " ne3=" << tensor->src[0]->ne[3] << " nb3=" << tensor->src[0]->nb[3] << std::endl; } if (tensor->src[1] != nullptr) { - std::cerr << "tensor->src[1]=" << tensor->src[1] << " name=" << tensor->src[1]->name << " op=" << ggml_op_name(tensor->src[1]->op) << " type=" << ggml_type_name(tensor->src[1]->type) << " backend=" << tensor->src[1]->backend << " ne0=" << tensor->src[1]->ne[0] << " nb0=" << tensor->src[1]->nb[0] << " ne1=" << tensor->src[1]->ne[1] << " nb1=" << tensor->src[1]->nb[1] << " ne2=" << tensor->src[1]->ne[2] << " nb2=" << tensor->src[1]->nb[2] << " ne3=" << tensor->src[1]->ne[3] << " nb3=" << tensor->src[1]->nb[3] << std::endl; + std::cerr << "tensor->src[1]=" << tensor->src[1] << " name=" << tensor->src[1]->name << " op=" << ggml_op_name(tensor->src[1]->op) << " type=" << ggml_type_name(tensor->src[1]->type) << " ne0=" << tensor->src[1]->ne[0] << " nb0=" << tensor->src[1]->nb[0] << " ne1=" << tensor->src[1]->ne[1] << " nb1=" << tensor->src[1]->nb[1] << " ne2=" << tensor->src[1]->ne[2] << " nb2=" << tensor->src[1]->nb[2] << " ne3=" << tensor->src[1]->ne[3] << " nb3=" << tensor->src[1]->nb[3] << std::endl; } std::cerr << std::endl << "Result:" << std::endl; ggml_vk_print_tensor_area(tensor, tensor_data, 5, 5, 0, 0); @@ -6577,43 +6632,11 @@ static void ggml_vk_print_tensor(ggml_backend_vk_context * ctx, const ggml_tenso std::vector done; ggml_vk_print_graph_origin(tensor, done); - if (tensor->backend == GGML_BACKEND_TYPE_GPU) { + if (ggml_backend_buffer_is_vk(tensor->buffer)) { free(tensor_data); } } -static void ggml_vk_check_tensor(const std::string& name, const ggml_tensor * tensor) { - return; - GGML_ASSERT(tensor->backend == GGML_BACKEND_TYPE_CPU); - if (tensor->type != GGML_TYPE_F32 && tensor->type != GGML_TYPE_F16) { - return; - } - for (int i3 = 0; i3 < tensor->ne[3]; i3++) { - for (int i2 = 0; i2 < tensor->ne[2]; i2++) { - for (int i1 = 0; i1 < tensor->ne[1]; i1++) { - for (int i0 = 0; i0 < tensor->ne[0]; i0++) { - float val = 0.0f; - if (tensor->type == GGML_TYPE_F32) { - val = *(float *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0]); - } else if (tensor->type == GGML_TYPE_F16) { - val = ggml_fp16_to_fp32(*(ggml_fp16_t *) ((char *) tensor->data + i3*tensor->nb[3] + i2*tensor->nb[2] + i1*tensor->nb[1] + i0*tensor->nb[0])); - } - if (std::isnan(val)) { - std::cerr << "ERROR: TENSOR CHECK " << name << ": Invalid value in " << ggml_op_name(tensor->op) << " i3=" << i3 << " i2=" << i2 << " i1=" << i1 << " i0=" << i0 << " val=" << val << std::endl; - std::cerr << "tensor=" << tensor << " tensor->type=" << ggml_type_name(tensor->type) << " tensor->backend: " << tensor->backend << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << std::endl; - std::cerr << std::endl; - ggml_vk_print_tensor_area(tensor, tensor->data, i0, i1, i2, i3); - std::cerr << std::endl; - std::vector done; - ggml_vk_print_graph_origin(tensor, done); - GGML_ASSERT(false); - } - } - } - } - } -} - void * comp_result; size_t comp_size; size_t comp_nb[GGML_MAX_DIMS]; @@ -6637,6 +6660,7 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ ggml_tensor * src0 = tensor->src[0]; ggml_tensor * src1 = tensor->src[1]; + ggml_tensor * src2 = tensor->src[2]; struct ggml_init_params iparams = { /*.mem_size =*/ 1024*1024*1024, @@ -6666,10 +6690,10 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ src0_buffer = malloc(src0_size); src0_clone->data = src0_buffer; - if (src0->backend == GGML_BACKEND_TYPE_CPU) { + if (ggml_backend_buffer_is_host(src0->buffer)) { memcpy(src0_clone->data, src0->data, src0_size); memcpy(src0_clone->nb, src0->nb, sizeof(size_t) * GGML_MAX_DIMS); - } else if (src0->backend == GGML_BACKEND_TYPE_GPU) { + } else if (ggml_backend_buffer_is_vk(src0->buffer)) { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src0->extra; vk_buffer buffer_gpu = extra->buffer_gpu.lock(); uint64_t offset = extra->offset; @@ -6700,8 +6724,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { ggml_vk_print_tensor(ctx, src0, "src0"); } - - ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src0", src0_clone); } if (src1 != nullptr) { src1_clone = ggml_dup_tensor(ggml_ctx, src1); @@ -6710,10 +6732,10 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ src1_buffer = malloc(src1_size); src1_clone->data = src1_buffer; - if (src1->backend == GGML_BACKEND_TYPE_CPU) { + if (ggml_backend_buffer_is_host(src1->buffer)) { memcpy(src1_clone->data, src1->data, src1_size); memcpy(src1_clone->nb, src1->nb, sizeof(size_t) * GGML_MAX_DIMS); - } else if (src1->backend == GGML_BACKEND_TYPE_GPU) { + } else if (ggml_backend_buffer_is_vk(src1->buffer)) { ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src1->extra; vk_buffer buffer_gpu = extra->buffer_gpu.lock(); uint64_t offset = extra->offset; @@ -6744,12 +6766,12 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { ggml_vk_print_tensor(ctx, src1, "src1"); std::cerr << "TENSOR CHECK: " << ggml_op_name(src1_clone->op) << " (check " << check_counter << ")" << std::endl; - std::cerr << "src1_clone=" << tensor << " src1_clone->backend: " << src1_clone->backend << " src1_clone->type: " << ggml_type_name(src1_clone->type) << " ne0=" << src1_clone->ne[0] << " nb0=" << src1_clone->nb[0] << " ne1=" << src1_clone->ne[1] << " nb1=" << src1_clone->nb[1] << " ne2=" << src1_clone->ne[2] << " nb2=" << src1_clone->nb[2] << " ne3=" << src1_clone->ne[3] << " nb3=" << src1_clone->nb[3] << std::endl; + std::cerr << "src1_clone=" << tensor << " src1_clone->type: " << ggml_type_name(src1_clone->type) << " ne0=" << src1_clone->ne[0] << " nb0=" << src1_clone->nb[0] << " ne1=" << src1_clone->ne[1] << " nb1=" << src1_clone->nb[1] << " ne2=" << src1_clone->ne[2] << " nb2=" << src1_clone->nb[2] << " ne3=" << src1_clone->ne[3] << " nb3=" << src1_clone->nb[3] << std::endl; if (src1->src[0] != nullptr) { - std::cerr << "src1->src[0]=" << src1->src[0] << " op=" << ggml_op_name(src1->src[0]->op) << " type=" << ggml_type_name(src1->src[0]->type) << " backend=" << src1->src[0]->backend << " ne0=" << src1->src[0]->ne[0] << " nb0=" << src1->src[0]->nb[0] << " ne1=" << src1->src[0]->ne[1] << " nb1=" << src1->src[0]->nb[1] << " ne2=" << src1->src[0]->ne[2] << " nb2=" << src1->src[0]->nb[2] << " ne3=" << src1->src[0]->ne[3] << " nb3=" << src1->src[0]->nb[3] << std::endl; + std::cerr << "src1->src[0]=" << src1->src[0] << " op=" << ggml_op_name(src1->src[0]->op) << " type=" << ggml_type_name(src1->src[0]->type) << " ne0=" << src1->src[0]->ne[0] << " nb0=" << src1->src[0]->nb[0] << " ne1=" << src1->src[0]->ne[1] << " nb1=" << src1->src[0]->nb[1] << " ne2=" << src1->src[0]->ne[2] << " nb2=" << src1->src[0]->nb[2] << " ne3=" << src1->src[0]->ne[3] << " nb3=" << src1->src[0]->nb[3] << std::endl; } if (src1->src[1] != nullptr) { - std::cerr << "src1->src[1]=" << src1->src[1] << " op=" << ggml_op_name(src1->src[1]->op) << " type=" << ggml_type_name(src1->src[1]->type) << " backend=" << src1->src[1]->backend << " ne0=" << src1->src[1]->ne[0] << " nb0=" << src1->src[1]->nb[0] << " ne1=" << src1->src[1]->ne[1] << " nb1=" << src1->src[1]->nb[1] << " ne2=" << src1->src[1]->ne[2] << " nb2=" << src1->src[1]->nb[2] << " ne3=" << src1->src[1]->ne[3] << " nb3=" << src1->src[1]->nb[3] << std::endl; + std::cerr << "src1->src[1]=" << src1->src[1] << " op=" << ggml_op_name(src1->src[1]->op) << " type=" << ggml_type_name(src1->src[1]->type) << " ne0=" << src1->src[1]->ne[0] << " nb0=" << src1->src[1]->nb[0] << " ne1=" << src1->src[1]->ne[1] << " nb1=" << src1->src[1]->nb[1] << " ne2=" << src1->src[1]->ne[2] << " nb2=" << src1->src[1]->nb[2] << " ne3=" << src1->src[1]->ne[3] << " nb3=" << src1->src[1]->nb[3] << std::endl; } std::cerr << std::endl << "Result:" << std::endl; ggml_vk_print_tensor_area(src1_clone, src1_clone->data, 5, 5, 0, 0); @@ -6760,8 +6782,64 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ std::vector done; ggml_vk_print_graph_origin(src1_clone, done); } + } + if (src2 != nullptr) { + src2_clone = ggml_dup_tensor(ggml_ctx, src2); - ggml_vk_check_tensor(std::string(ggml_op_name(tensor->op)) + "->src1", src1_clone); + src2_size = ggml_nbytes(src2); + + src2_buffer = malloc(src2_size); + src2_clone->data = src2_buffer; + if (ggml_backend_buffer_is_host(src2->buffer)) { + memcpy(src2_clone->data, src2->data, src2_size); + memcpy(src2_clone->nb, src2->nb, sizeof(size_t) * GGML_MAX_DIMS); + } else if (ggml_backend_buffer_is_vk(src2->buffer)) { + ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) src2->extra; + vk_buffer buffer_gpu = extra->buffer_gpu.lock(); + uint64_t offset = extra->offset; + if (!ggml_is_contiguous(src2) && ggml_vk_dim01_contiguous(src2)) { + for (int i3 = 0; i3 < src2->ne[3]; i3++) { + for (int i2 = 0; i2 < src2->ne[2]; i2++) { + const int idx = i3*src2->ne[2] + i2; + ggml_vk_buffer_read(ctx, buffer_gpu, offset + idx * src2->nb[2], ((char *)src2_clone->data + idx * src2_clone->nb[2]), src2->ne[1] * src2->nb[1]); + } + } + + src2_clone->nb[0] = src2->nb[0]; + src2_clone->nb[1] = src2->nb[1]; + for (int i = 2; i < GGML_MAX_DIMS; i++) { + src2_clone->nb[i] = src2_clone->nb[i - 1]*src2_clone->ne[i - 1]; + } + } else { + if (offset + src2_size >= buffer_gpu->size) { + src2_size = buffer_gpu->size - offset; + } + ggml_vk_buffer_read(ctx, buffer_gpu, offset, src2_clone->data, src2_size); + memcpy(src2_clone->nb, src2->nb, sizeof(size_t) * GGML_MAX_DIMS); + } + } else { + GGML_ASSERT(false); + } + + if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { + ggml_vk_print_tensor(ctx, src2, "src2"); + std::cerr << "TENSOR CHECK: " << ggml_op_name(src2_clone->op) << " (check " << check_counter << ")" << std::endl; + std::cerr << "src2_clone=" << tensor << " src2_clone->type: " << ggml_type_name(src2_clone->type) << " ne0=" << src2_clone->ne[0] << " nb0=" << src2_clone->nb[0] << " ne1=" << src2_clone->ne[1] << " nb1=" << src2_clone->nb[1] << " ne2=" << src2_clone->ne[2] << " nb2=" << src2_clone->nb[2] << " ne3=" << src2_clone->ne[3] << " nb3=" << src2_clone->nb[3] << std::endl; + if (src2->src[0] != nullptr) { + std::cerr << "src2->src[0]=" << src2->src[0] << " op=" << ggml_op_name(src2->src[0]->op) << " type=" << ggml_type_name(src2->src[0]->type) << " ne0=" << src2->src[0]->ne[0] << " nb0=" << src2->src[0]->nb[0] << " ne1=" << src2->src[0]->ne[1] << " nb1=" << src2->src[0]->nb[1] << " ne2=" << src2->src[0]->ne[2] << " nb2=" << src2->src[0]->nb[2] << " ne3=" << src2->src[0]->ne[3] << " nb3=" << src2->src[0]->nb[3] << std::endl; + } + if (src2->src[1] != nullptr) { + std::cerr << "src2->src[1]=" << src2->src[1] << " op=" << ggml_op_name(src2->src[1]->op) << " type=" << ggml_type_name(src2->src[1]->type) << " ne0=" << src2->src[1]->ne[0] << " nb0=" << src2->src[1]->nb[0] << " ne1=" << src2->src[1]->ne[1] << " nb1=" << src2->src[1]->nb[1] << " ne2=" << src2->src[1]->ne[2] << " nb2=" << src2->src[1]->nb[2] << " ne3=" << src2->src[1]->ne[3] << " nb3=" << src2->src[1]->nb[3] << std::endl; + } + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(src2_clone, src2_clone->data, 5, 5, 0, 0); + std::cerr << std::endl; + std::cerr << std::endl << "Result:" << std::endl; + ggml_vk_print_tensor_area(src2_clone, src2_clone->data, 5, 5, 1, 0); + std::cerr << std::endl; + std::vector done; + ggml_vk_print_graph_origin(src2_clone, done); + } } if (tensor->op == GGML_OP_MUL_MAT) { @@ -6799,7 +6877,7 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ float attn_factor = ((float *) tensor->op_params)[8]; float beta_fast = ((float *) tensor->op_params)[9]; float beta_slow = ((float *) tensor->op_params)[10]; - tensor_clone = ggml_rope_custom(ggml_ctx, src0_clone, src1_clone, n_dims, mode, n_ggml_ctx, n_orig_ggml_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + tensor_clone = ggml_rope_ext(ggml_ctx, src0_clone, src1_clone, src2_clone, n_dims, mode, n_ggml_ctx, n_orig_ggml_ctx, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); } else if (tensor->op == GGML_OP_UNARY) { switch (ggml_get_unary_op(tensor)) { case GGML_UNARY_OP_SILU: @@ -6847,7 +6925,6 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_compute_ ggml_graph_compute_with_ctx(ggml_ctx, cgraph, 8); - ggml_vk_check_tensor(ggml_op_name(tensor->op), tensor_clone); if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { ggml_vk_print_tensor(ctx, tensor_clone, "tensor_clone"); } @@ -6888,7 +6965,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ void * tensor_data = tensor->data; - if (tensor->backend == GGML_BACKEND_TYPE_GPU) { + if (ggml_backend_buffer_is_vk(tensor->buffer)) { size_t tensor_size = ggml_nbytes(tensor); tensor_data = malloc(tensor_size); @@ -6936,12 +7013,12 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ if ((std::isnan(correct) != std::isnan(result)) || (std::isinf(correct) != std::isinf(result)) || !buffer_size_fit) { std::cerr << "ERROR: Invalid value in " << ggml_op_name(tensor->op) << " i3=" << i3 << " i2=" << i2 << " i1=" << i1 << " i0=" << i0 << " result=" << result << " correct=" << correct << " avg_err=" << (avg_err / counter) << std::endl; - std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->backend: " << tensor->backend << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; + std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; if (src0 != nullptr) { - std::cerr << "src0=" << src0 << " src0->name=" << src0->name << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " backend=" << src0->backend << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; + std::cerr << "src0=" << src0 << " src0->name=" << src0->name << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; } if (src1 != nullptr) { - std::cerr << "src1=" << src1 << " src1->name=" << src1->name << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " backend=" << src1->backend << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; + std::cerr << "src1=" << src1 << " src1->name=" << src1->name << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; } std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; std::cerr << std::endl << "Result:" << std::endl; @@ -6977,12 +7054,12 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ if (vk_output_tensor > 0 && vk_output_tensor == check_counter) { std::cerr << "TENSOR CHECK: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl; - std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->backend: " << tensor->backend << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; + std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; if (src0 != nullptr) { - std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " backend=" << src0->backend << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; + std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; } if (src1 != nullptr) { - std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " backend=" << src1->backend << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; + std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; } std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; std::cerr << std::endl << "Result:" << std::endl; @@ -7001,12 +7078,12 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ if (avg_err > 0.05 || std::isnan(avg_err)) { std::cerr << "ERROR: avg_err=" << avg_err << " in " << ggml_op_name(tensor->op) << " (check " << check_counter << ")" << std::endl; - std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->backend: " << tensor->backend << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; + std::cerr << "tensor=" << tensor << " tensor->name=" << tensor->name << " tensor->type: " << ggml_type_name(tensor->type) << " ne0=" << tensor->ne[0] << " nb0=" << tensor->nb[0] << " ne1=" << tensor->ne[1] << " nb1=" << tensor->nb[1] << " ne2=" << tensor->ne[2] << " nb2=" << tensor->nb[2] << " ne3=" << tensor->ne[3] << " nb3=" << tensor->nb[3] << " offset=" << tensor->view_offs << std::endl; if (src0 != nullptr) { - std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " backend=" << src0->backend << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; + std::cerr << "src0=" << src0 << " op=" << ggml_op_name(src0->op) << " type=" << ggml_type_name(src0->type) << " ne0=" << src0->ne[0] << " nb0=" << src0->nb[0] << " ne1=" << src0->ne[1] << " nb1=" << src0->nb[1] << " ne2=" << src0->ne[2] << " nb2=" << src0->nb[2] << " ne3=" << src0->ne[3] << " nb3=" << src0->nb[3] << " offset=" << src0->view_offs << std::endl; } if (src1 != nullptr) { - std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " backend=" << src1->backend << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; + std::cerr << "src1=" << src1 << " op=" << ggml_op_name(src1->op) << " type=" << ggml_type_name(src1->type) << " ne0=" << src1->ne[0] << " nb0=" << src1->nb[0] << " ne1=" << src1->ne[1] << " nb1=" << src1->nb[1] << " ne2=" << src1->ne[2] << " nb2=" << src1->nb[2] << " ne3=" << src1->ne[3] << " nb3=" << src1->nb[3] << " offset=" << src1->view_offs << std::endl; } std::cerr << "First error: result=" << first_error_result << " correct=" << first_error_correct << " i3=" << first_error[3] << " i2=" << first_error[2] << " i1=" << first_error[1] << " i0=" << first_error[0] << std::endl; std::cerr << std::endl << "Result:" << std::endl; @@ -7018,14 +7095,14 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_compute_ ggml_vk_print_graph_origin(tensor, done); GGML_ASSERT(false); } else { - std::cerr << check_counter << " " << tensor->name << " op=" << ggml_op_name(tensor->op) << " backend=" << tensor->backend << " avg_err=" << avg_err << std::endl; + std::cerr << check_counter << " " << tensor->name << " op=" << ggml_op_name(tensor->op) << " avg_err=" << avg_err << std::endl; } free(comp_result); comp_result = nullptr; comp_size = 0; - if (tensor->backend == GGML_BACKEND_TYPE_GPU) { + if (ggml_backend_buffer_is_vk(tensor->buffer)) { free(tensor_data); } } diff --git a/ggml_vk_generate_shaders.py b/ggml_vk_generate_shaders.py index 8096c03b7..a8f7373df 100644 --- a/ggml_vk_generate_shaders.py +++ b/ggml_vk_generate_shaders.py @@ -2609,7 +2609,8 @@ layout(local_size_x = 1, local_size_y = 256, local_size_z = 1) in; layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; layout (binding = 1) readonly buffer Y {int data_b[];}; -layout (binding = 2) writeonly buffer D {D_TYPE data_d[];}; +layout (binding = 2) readonly buffer Z {float data_freq_factors[];}; +layout (binding = 3) writeonly buffer D {D_TYPE data_d[];}; layout (push_constant) uniform parameter { uint ncols; @@ -2622,6 +2623,7 @@ layout (push_constant) uniform parameter { float corr_dims[4]; float theta_scale; float inv_ndims; + uint has_freq_facs; } p; float rope_yarn_ramp(const float low, const float high, const uint i0) { @@ -2671,7 +2673,8 @@ void main() { const float cur_rot = p.inv_ndims * ic - ib; const int pos = data_b[i2]; - const float theta_base = pos*p.freq_scale*pow(p.theta_scale, col/2.0f); + const float freq_factor = p.has_freq_facs != 0 ? data_freq_factors[ic/2] : 1.0f; + const float theta_base = pos*p.freq_scale*pow(p.theta_scale, col/2.0f) / freq_factor; float cos_theta, sin_theta; rope_yarn(theta_base, uint(cur_rot), cos_theta, sin_theta); From e84b71c2c6da6e69c8f815168ea836f9716a325e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 23 May 2024 10:00:21 +0300 Subject: [PATCH 114/137] ggml : drop support for QK_K=64 (#7473) * ggml : drop support for QK_K=64 ggml-ci * opencl : restore QK_K=256 define --- CMakeLists.txt | 5 - Makefile | 4 - ci/run.sh | 3 +- ggml-common.h | 54 - ggml-cuda/convert.cu | 138 -- ggml-cuda/dmmv.cu | 151 --- ggml-cuda/mmq.cu | 6 - ggml-cuda/vecdotq.cuh | 126 -- ggml-metal.m | 17 - ggml-metal.metal | 398 +----- ggml-opencl.cpp | 2 +- ggml-quants.c | 2672 +------------------------------------ ggml-sycl.cpp | 472 +------ ggml.c | 12 - gguf-py/gguf/constants.py | 3 +- llama.cpp | 12 +- 16 files changed, 26 insertions(+), 4049 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c09d834fb..ef02ff669 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,6 @@ set(LLAMA_METAL_MACOSX_VERSION_MIN "" CACHE STRING set(LLAMA_METAL_STD "" CACHE STRING "llama: metal standard version (-std flag)") option(LLAMA_KOMPUTE "llama: use Kompute" OFF) option(LLAMA_RPC "llama: use RPC" OFF) -option(LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF) option(LLAMA_SYCL "llama: use SYCL" OFF) option(LLAMA_SYCL_F16 "llama: use 16 bit floats for sycl calculations" OFF) set(LLAMA_SYCL_TARGET "INTEL" CACHE STRING "llama: sycl target device") @@ -384,10 +383,6 @@ if (LLAMA_LLAMAFILE) set(GGML_SOURCES_LLAMAFILE sgemm.cpp) endif() -if (LLAMA_QKK_64) - add_compile_definitions(GGML_QKK_64) -endif() - if (LLAMA_CUBLAS) message(WARNING "LLAMA_CUBLAS is deprecated and will be removed in the future.\nUse LLAMA_CUDA instead") set(LLAMA_CUDA ON) diff --git a/Makefile b/Makefile index 6b7c853b3..fe63cbd60 100644 --- a/Makefile +++ b/Makefile @@ -389,10 +389,6 @@ else MK_CXXFLAGS += -march=rv64gcv -mabi=lp64d endif -ifdef LLAMA_QKK_64 - MK_CPPFLAGS += -DGGML_QKK_64 -endif - ifndef LLAMA_NO_ACCELERATE # Mac OS - include Accelerate framework. # `-framework Accelerate` works both with Apple Silicon and Mac Intel diff --git a/ci/run.sh b/ci/run.sh index d5972480b..79dcd0772 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -606,7 +606,8 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then if [ -z ${GG_BUILD_VRAM_GB} ] || [ ${GG_BUILD_VRAM_GB} -ge 8 ]; then if [ -z ${GG_BUILD_CUDA} ]; then - test $ret -eq 0 && gg_run open_llama_3b_v2 + #test $ret -eq 0 && gg_run open_llama_3b_v2 + date # dummy else test $ret -eq 0 && gg_run open_llama_7b_v2 fi diff --git a/ggml-common.h b/ggml-common.h index 43c7978a0..77e6bfba4 100644 --- a/ggml-common.h +++ b/ggml-common.h @@ -65,13 +65,8 @@ typedef sycl::half2 ggml_half2; // QK = number of values after dequantization // QK_K = super-block size -#ifdef GGML_QKK_64 -#define QK_K 64 -#define K_SCALE_SIZE 4 -#else #define QK_K 256 #define K_SCALE_SIZE 12 -#endif // GGML_QKK_64 #if defined(GGML_COMMON_DECL_CUDA) || defined(GGML_COMMON_DECL_HIP) || defined(GGML_COMMON_DECL_SYCL) // QR = QK / number of values before dequantization @@ -131,13 +126,8 @@ typedef sycl::half2 ggml_half2; #define QI4_NL (QK4_NL / (4*QR4_NL)) #define QR4_NL 2 -#if QK_K == 64 -#define QI4_XS QI4_NL -#define QR4_XS QR4_NL -#else #define QI4_XS (QK_K / (4*QR4_XS)) #define QR4_XS 8 -#endif #endif // GGML_COMMON_DECL_CUDA || GGML_COMMON_DECL_HIP @@ -228,15 +218,6 @@ static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wro // weight is represented as x = a * q // 16 blocks of 16 elements each // Effectively 3.4375 bits per weight -#ifdef GGML_QKK_64 -typedef struct { - uint8_t hmask[QK_K/8]; // quants - high bit - uint8_t qs[QK_K/4]; // quants - low 2 bits - uint8_t scales[2]; - ggml_half d; // super-block scale -} block_q3_K; -static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 2, "wrong q3_K block size/padding"); -#else typedef struct { uint8_t hmask[QK_K/8]; // quants - high bit uint8_t qs[QK_K/4]; // quants - low 2 bits @@ -244,20 +225,11 @@ typedef struct { ggml_half d; // super-block scale } block_q3_K; static_assert(sizeof(block_q3_K) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8 + 12, "wrong q3_K block size/padding"); -#endif // 4-bit quantization // 8 blocks of 32 elements each // weight is represented as x = a * q + b // Effectively 4.5 bits per weight -#ifdef GGML_QKK_64 -typedef struct { - ggml_half d[2]; // super-block scales/mins - uint8_t scales[2]; // 4-bit block scales/mins - uint8_t qs[QK_K/2]; // 4--bit quants -} block_q4_K; -static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + QK_K/2 + 2, "wrong q4_K block size/padding"); -#else typedef struct { union { struct { @@ -270,21 +242,11 @@ typedef struct { uint8_t qs[QK_K/2]; // 4--bit quants } block_q4_K; static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding"); -#endif // 5-bit quantization // 8 blocks of 32 elements each // weight is represented as x = a * q + b // Effectively 5.5 bits per weight -#ifdef GGML_QKK_64 -typedef struct { - ggml_half d; // super-block scale - int8_t scales[QK_K/16]; // 8-bit block scales - uint8_t qh[QK_K/8]; // quants, high bit - uint8_t qs[QK_K/2]; // quants, low 4 bits -} block_q5_K; -static_assert(sizeof(block_q5_K) == sizeof(ggml_half) + QK_K/2 + QK_K/8 + QK_K/16, "wrong q5_K block size/padding"); -#else typedef struct { union { struct { @@ -298,7 +260,6 @@ typedef struct { uint8_t qs[QK_K/2]; // quants, low 4 bits } block_q5_K; static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2 + QK_K/8, "wrong q5_K block size/padding"); -#endif // 6-bit quantization // weight is represented as x = a * q @@ -356,11 +317,7 @@ typedef struct { static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_half) + 3*(QK_K/8), "wrong iq3_xxs block size/padding"); // 3.4375 bpw -#if QK_K == 64 -#define IQ3S_N_SCALE 2 -#else #define IQ3S_N_SCALE QK_K/64 -#endif typedef struct { ggml_half d; uint8_t qs[QK_K/4]; @@ -381,16 +338,9 @@ static_assert(sizeof(block_iq1_s) == sizeof(ggml_half) + QK_K/8 + QK_K/16, "wron typedef struct { uint8_t qs[QK_K/8]; // grid index, low 8 bits uint8_t qh[QK_K/16]; // grid index, high 3 bits + grid shift bit (for two groups of 8) -#if QK_K == 64 - ggml_half d; -#endif uint8_t scales[QK_K/32]; // 3-bit block scales (4-bit if QK_K == 64) } block_iq1_m; -#if QK_K == 64 -static_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32 + sizeof(ggml_half), "wrong iq1_m block size/padding"); -#else static_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, "wrong iq1_m block size/padding"); -#endif // Used by IQ1_M quants typedef union { @@ -406,9 +356,6 @@ typedef struct { } block_iq4_nl; static_assert(sizeof(block_iq4_nl) == sizeof(ggml_half) + QK4_NL/2, "wrong iq4_nl block size/padding"); -#if QK_K == 64 -#define block_iq4_xs block_iq4_nl -#else typedef struct { ggml_half d; uint16_t scales_h; @@ -416,7 +363,6 @@ typedef struct { uint8_t qs[QK_K/2]; } block_iq4_xs; static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); -#endif #endif // GGML_COMMON_DECL #endif // GGML_COMMON_DECL diff --git a/ggml-cuda/convert.cu b/ggml-cuda/convert.cu index 830e2d756..c0a444707 100644 --- a/ggml-cuda/convert.cu +++ b/ggml-cuda/convert.cu @@ -131,7 +131,6 @@ static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t const block_q2_K * x = (const block_q2_K *) vx; const int64_t tid = threadIdx.x; -#if QK_K == 256 const int64_t n = tid/32; const int64_t l = tid - 32*n; const int64_t is = 8*n + l/16; @@ -145,17 +144,6 @@ static __global__ void dequantize_block_q2_K(const void * __restrict__ vx, dst_t y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4); y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4); y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4); -#else - const int64_t is = tid/16; // 0 or 1 - const int64_t il = tid%16; // 0...15 - const uint8_t q = x[i].qs[il] >> (2*is); - dst_t * y = yy + i*QK_K + 16*is + il; - float dall = __low2half(x[i].dm); - float dmin = __high2half(x[i].dm); - y[ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4); - y[32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+2] >> 4); -#endif - } template @@ -164,7 +152,6 @@ static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t const int64_t i = blockIdx.x; const block_q3_K * x = (const block_q3_K *) vx; -#if QK_K == 256 const int64_t r = threadIdx.x/4; const int64_t tid = r/2; const int64_t is0 = r%2; @@ -188,31 +175,8 @@ static __global__ void dequantize_block_q3_K(const void * __restrict__ vx, dst_t const uint8_t * hm = x[i].hmask; for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)); -#else - const int64_t tid = threadIdx.x; - const int64_t is = tid/16; // 0 or 1 - const int64_t il = tid%16; // 0...15 - const int64_t im = il/8; // 0...1 - const int64_t in = il%8; // 0...7 - - dst_t * y = yy + i*QK_K + 16*is + il; - - const uint8_t q = x[i].qs[il] >> (2*is); - const uint8_t h = x[i].hmask[in] >> (2*is + im); - const float d = (float)x[i].d; - - if (is == 0) { - y[ 0] = d * ((x[i].scales[0] & 0xF) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4)); - y[32] = d * ((x[i].scales[1] & 0xF) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4)); - } else { - y[ 0] = d * ((x[i].scales[0] >> 4) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4)); - y[32] = d * ((x[i].scales[1] >> 4) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4)); - } -#endif - } -#if QK_K == 256 static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) { if (j < 4) { d = q[j] & 63; m = q[j + 4] & 63; @@ -221,7 +185,6 @@ static inline __device__ void get_scale_min_k4(int j, const uint8_t * q, uint8_t m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); } } -#endif template static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy) { @@ -229,7 +192,6 @@ static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t const int64_t i = blockIdx.x; -#if QK_K == 256 // assume 32 threads const int64_t tid = threadIdx.x; const int64_t il = tid/8; @@ -253,15 +215,6 @@ static __global__ void dequantize_block_q4_K(const void * __restrict__ vx, dst_t y[l + 0] = d1 * (q[l] & 0xF) - m1; y[l +32] = d2 * (q[l] >> 4) - m2; } -#else - const int64_t tid = threadIdx.x; - const uint8_t * q = x[i].qs; - dst_t * y = yy + i*QK_K; - const float d = (float)x[i].dm[0]; - const float m = (float)x[i].dm[1]; - y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4); - y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4); -#endif } template @@ -270,7 +223,6 @@ static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t const int64_t i = blockIdx.x; -#if QK_K == 256 // assume 64 threads - this is very slightly better than the one below const int64_t tid = threadIdx.x; const int64_t il = tid/16; // il is in 0...3 @@ -297,18 +249,6 @@ static __global__ void dequantize_block_q5_K(const void * __restrict__ vx, dst_t hm <<= 1; y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2; y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2; -#else - const int64_t tid = threadIdx.x; - const uint8_t q = x[i].qs[tid]; - const int64_t im = tid/8; // 0...3 - const int64_t in = tid%8; // 0...7 - const int64_t is = tid/16; // 0 or 1 - const uint8_t h = x[i].qh[in] >> im; - const float d = x[i].d; - dst_t * y = yy + i*QK_K + tid; - y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16)); - y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16)); -#endif } template @@ -316,7 +256,6 @@ static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t const block_q6_K * x = (const block_q6_K *) vx; const int64_t i = blockIdx.x; -#if QK_K == 256 // assume 64 threads - this is very slightly better than the one below const int64_t tid = threadIdx.x; @@ -336,24 +275,6 @@ static __global__ void dequantize_block_q6_K(const void * __restrict__ vx, dst_t y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32); y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32); y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32); -#else - - // assume 32 threads - const int64_t tid = threadIdx.x; - const int64_t ip = tid/16; // 0 or 1 - const int64_t il = tid - 16*ip; // 0...15 - - dst_t * y = yy + i*QK_K + 16*ip + il; - - const float d = x[i].d; - - const uint8_t ql = x[i].ql[16*ip + il]; - const uint8_t qh = x[i].qh[il] >> (2*ip); - const int8_t * sc = x[i].scales; - - y[ 0] = d * sc[ip+0] * ((int8_t)((ql & 0xF) | (((qh >> 0) & 3) << 4)) - 32); - y[32] = d * sc[ip+2] * ((int8_t)((ql >> 4) | (((qh >> 4) & 3) << 4)) - 32); -#endif } template @@ -363,7 +284,6 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, ds const block_iq2_xxs * x = (const block_iq2_xxs *) vx; const int64_t tid = threadIdx.x; -#if QK_K == 256 const int64_t il = tid/8; // 0...3 const int64_t ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -374,10 +294,6 @@ static __global__ void dequantize_block_iq2_xxs(const void * __restrict__ vx, ds const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f; const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*il) & 127]; for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); -#else - NO_DEVICE_CODE; -#endif - } template @@ -387,7 +303,6 @@ static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst const block_iq2_xs * x = (const block_iq2_xs *) vx; const int64_t tid = threadIdx.x; -#if QK_K == 256 const int64_t il = tid/8; // 0...3 const int64_t ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -396,10 +311,6 @@ static __global__ void dequantize_block_iq2_xs(const void * __restrict__ vx, dst const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); -#else - NO_DEVICE_CODE; -#endif - } template @@ -409,7 +320,6 @@ static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_ const block_iq2_s * x = (const block_iq2_s *) vx; const int64_t tid = threadIdx.x; -#if QK_K == 256 const int64_t il = tid/8; // 0...3 const int64_t ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -417,10 +327,6 @@ static __global__ void dequantize_block_iq2_s(const void * __restrict__ vx, dst_ const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; const uint8_t signs = x[i].qs[QK_K/8+4*ib+il]; for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); -#else - NO_DEVICE_CODE; -#endif - } template @@ -430,7 +336,6 @@ static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, ds const block_iq3_xxs * x = (const block_iq3_xxs *) vx; const int64_t tid = threadIdx.x; -#if QK_K == 256 const int64_t il = tid/8; // 0...3 const int64_t ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -445,10 +350,6 @@ static __global__ void dequantize_block_iq3_xxs(const void * __restrict__ vx, ds y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); } -#else - NO_DEVICE_CODE; -#endif - } template @@ -458,7 +359,6 @@ static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_ const block_iq3_s * x = (const block_iq3_s *) vx; const int64_t tid = threadIdx.x; -#if QK_K == 256 const int64_t il = tid/8; // 0...3 const int64_t ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -471,10 +371,6 @@ static __global__ void dequantize_block_iq3_s(const void * __restrict__ vx, dst_ y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); } -#else - NO_DEVICE_CODE; -#endif - } template @@ -484,7 +380,6 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_ const block_iq1_s * x = (const block_iq1_s *) vx; const int64_t tid = threadIdx.x; -#if QK_K == 256 const int64_t il = tid/8; // 0...3 const int64_t ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -497,10 +392,6 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_ for (int j = 0; j < 8; ++j) { y[j] = d * (q[j] + delta); } -#else - NO_DEVICE_CODE; -#endif - } template @@ -510,7 +401,6 @@ static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_ const block_iq1_m * x = (const block_iq1_m *) vx; const int64_t tid = threadIdx.x; -#if QK_K == 256 const int64_t il = tid/8; // 0...3 const int64_t ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -527,13 +417,8 @@ static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_ for (int j = 0; j < 8; ++j) { y[j] = d * (q[j] + delta); } -#else - NO_DEVICE_CODE; -#endif - } - template static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst_t * __restrict__ yy) { @@ -550,10 +435,8 @@ static __global__ void dequantize_block_iq4_nl(const void * __restrict__ vx, dst y[j+ 0] = d * kvalues_iq4nl[q4[j] & 0xf]; y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; } - } -#if QK_K != 64 template static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) { const int64_t i = blockIdx.x; @@ -570,7 +453,6 @@ static __global__ void dequantize_block_iq4_xs(const void * __restrict__ vx, dst y[j+16] = d * kvalues_iq4nl[q4[j] >> 4]; } } -#endif template static void dequantize_block_cuda(const void * __restrict__ vx, dst_t * __restrict__ y, const int64_t k, cudaStream_t stream) { @@ -592,21 +474,13 @@ static void dequantize_block_q8_0_f16_cuda(const void * __restrict__ vx, half * template static void dequantize_row_q2_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { const int nb = k / QK_K; -#if QK_K == 256 dequantize_block_q2_K<<>>(vx, y); -#else - dequantize_block_q2_K<<>>(vx, y); -#endif } template static void dequantize_row_q3_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { const int nb = k / QK_K; -#if QK_K == 256 dequantize_block_q3_K<<>>(vx, y); -#else - dequantize_block_q3_K<<>>(vx, y); -#endif } template @@ -632,21 +506,13 @@ static void dequantize_row_q4_K_cuda(const void * vx, dst_t * y, const int64_t k template static void dequantize_row_q5_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { const int nb = k / QK_K; -#if QK_K == 256 dequantize_block_q5_K<<>>(vx, y); -#else - dequantize_block_q5_K<<>>(vx, y); -#endif } template static void dequantize_row_q6_K_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { const int nb = k / QK_K; -#if QK_K == 256 dequantize_block_q6_K<<>>(vx, y); -#else - dequantize_block_q6_K<<>>(vx, y); -#endif } template @@ -700,11 +566,7 @@ static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t template static void dequantize_row_iq4_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { const int nb = (k + QK_K - 1) / QK_K; -#if QK_K == 64 - dequantize_block_iq4_nl<<>>(vx, y); -#else dequantize_block_iq4_xs<<>>(vx, y); -#endif } template diff --git a/ggml-cuda/dmmv.cu b/ggml-cuda/dmmv.cu index 7313e3e17..47d4d5d9e 100644 --- a/ggml-cuda/dmmv.cu +++ b/ggml-cuda/dmmv.cu @@ -22,7 +22,6 @@ static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, float tmp = 0; // partial sum for thread in warp -#if QK_K == 256 const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...15 const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0,1 @@ -71,37 +70,6 @@ static __global__ void dequantize_mul_mat_vec_q2_k(const void * __restrict__ vx, tmp += dall * sum1 - dmin * sum2; } -#else - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3 - const int offset = tid * K_QUANTS_PER_ITERATION; - - uint32_t uaux[2]; - const uint8_t * d = (const uint8_t *)uaux; - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - - const float * y = yy + i * QK_K + offset; - const uint8_t * q = x[i].qs + offset; - const uint32_t * s = (const uint32_t *)x[i].scales; - - uaux[0] = s[0] & 0x0f0f0f0f; - uaux[1] = (s[0] >> 4) & 0x0f0f0f0f; - - const float2 dall = __half22float2(x[i].dm); - - float sum1 = 0, sum2 = 0; - for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) { - const uint8_t ql = q[l]; - sum1 += y[l+ 0] * d[0] * ((ql >> 0) & 3) - + y[l+16] * d[1] * ((ql >> 2) & 3) - + y[l+32] * d[2] * ((ql >> 4) & 3) - + y[l+48] * d[3] * ((ql >> 6) & 3); - sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7]; - } - tmp += dall.x * sum1 - dall.y * sum2; - } -#endif // sum up partial sums and write back result tmp = warp_reduce_sum(tmp); @@ -123,8 +91,6 @@ static __global__ void dequantize_mul_mat_vec_q3_k(const void * __restrict__ vx, float tmp = 0; // partial sum for thread in warp -#if QK_K == 256 - const uint16_t kmask1 = 0x0303; const uint16_t kmask2 = 0x0f0f; @@ -175,34 +141,6 @@ static __global__ void dequantize_mul_mat_vec_q3_k(const void * __restrict__ vx, tmp += d * sum; } -#else - - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3 - const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14 - const int in = offset/8; // 0 or 1 - const int im = offset%8; // 0...7 - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - - const float * y = yy + i * QK_K + offset; - const uint8_t * q = x[i].qs + offset; - const uint8_t * s = x[i].scales; - - const float dall = (float)x[i].d; - - float sum = 0; - for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) { - const uint8_t hl = x[i].hmask[im+l] >> in; - const uint8_t ql = q[l]; - sum += y[l+ 0] * dall * ((s[0] & 0xF) - 8) * ((int8_t)((ql >> 0) & 3) - ((hl >> 0) & 1 ? 0 : 4)) - + y[l+16] * dall * ((s[0] >> 4) - 8) * ((int8_t)((ql >> 2) & 3) - ((hl >> 2) & 1 ? 0 : 4)) - + y[l+32] * dall * ((s[1] & 0xF) - 8) * ((int8_t)((ql >> 4) & 3) - ((hl >> 4) & 1 ? 0 : 4)) - + y[l+48] * dall * ((s[1] >> 4) - 8) * ((int8_t)((ql >> 6) & 3) - ((hl >> 6) & 1 ? 0 : 4)); - } - tmp += sum; - } -#endif // sum up partial sums and write back result tmp = warp_reduce_sum(tmp); @@ -221,7 +159,6 @@ static __global__ void dequantize_mul_mat_vec_q4_k(const void * __restrict__ vx, const block_q4_K * x = (const block_q4_K *)vx + ib0; -#if QK_K == 256 const uint16_t kmask1 = 0x3f3f; const uint16_t kmask2 = 0x0f0f; const uint16_t kmask3 = 0xc0c0; @@ -306,36 +243,6 @@ static __global__ void dequantize_mul_mat_vec_q4_k(const void * __restrict__ vx, #endif } -#else - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); - - const int step = tid * K_QUANTS_PER_ITERATION; - - uint16_t aux16[2]; - const uint8_t * s = (const uint8_t *)aux16; - - float tmp = 0; - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - const uint8_t * q = x[i].qs + step; - const float * y = yy + i*QK_K + step; - const uint16_t * a = (const uint16_t *)x[i].scales; - aux16[0] = a[0] & 0x0f0f; - aux16[1] = (a[0] >> 4) & 0x0f0f; - const float d = (float)x[i].dm[0]; - const float m = (float)x[i].dm[1]; - float sum = 0.f; - for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) { - sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2]) - + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2]) - + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3]) - + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]); - } - tmp += sum; - } - -#endif // sum up partial sums and write back result tmp = warp_reduce_sum(tmp); @@ -355,7 +262,6 @@ static __global__ void dequantize_mul_mat_vec_q5_k(const void * __restrict__ vx, float tmp = 0; // partial sum for thread in warp -#if QK_K == 256 const uint16_t kmask1 = 0x3f3f; const uint16_t kmask2 = 0x0f0f; const uint16_t kmask3 = 0xc0c0; @@ -426,30 +332,6 @@ static __global__ void dequantize_mul_mat_vec_q5_k(const void * __restrict__ vx, tmp += dall * (sum.x * sc[0] + sum.y * sc[1] + sum.z * sc[4] + sum.w * sc[5]) - dmin * smin; } -#else - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...15 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); - const int step = tid * K_QUANTS_PER_ITERATION; - const int im = step/8; - const int in = step%8; - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - const uint8_t * q = x[i].qs + step; - const int8_t * s = x[i].scales; - const float * y = yy + i*QK_K + step; - const float d = x[i].d; - float sum = 0.f; - for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) { - const uint8_t h = x[i].qh[in+j] >> im; - sum += y[j+ 0] * d * s[0] * ((q[j+ 0] & 0xF) - ((h >> 0) & 1 ? 0 : 16)) - + y[j+16] * d * s[1] * ((q[j+16] & 0xF) - ((h >> 2) & 1 ? 0 : 16)) - + y[j+32] * d * s[2] * ((q[j+ 0] >> 4) - ((h >> 4) & 1 ? 0 : 16)) - + y[j+48] * d * s[3] * ((q[j+16] >> 4) - ((h >> 6) & 1 ? 0 : 16)); - } - tmp += sum; - } -#endif - // sum up partial sums and write back result tmp = warp_reduce_sum(tmp); @@ -470,8 +352,6 @@ static __global__ void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const block_q6_K * x = (const block_q6_K *)vx + ib0; -#if QK_K == 256 - const int tid = threadIdx.x/K_QUANTS_PER_ITERATION; // 0...31 or 0...16 const int ix = threadIdx.x%K_QUANTS_PER_ITERATION; // 0 or 0, 1 @@ -526,37 +406,6 @@ static __global__ void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, } -#else - - const int tid = threadIdx.x/(2*K_QUANTS_PER_ITERATION); // 0...7 - const int ix = threadIdx.x%(2*K_QUANTS_PER_ITERATION); // 0...3 - - const int step = tid * K_QUANTS_PER_ITERATION; - - float tmp = 0; // partial sum for thread in warp - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - - const float * y = yy + i * QK_K + step; - const uint8_t * ql = x[i].ql + step; - const uint8_t * qh = x[i].qh + step; - const int8_t * s = x[i].scales; - - const float d = x[i+0].d; - - float sum = 0; - for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) { - sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32) - + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32) - + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32) - + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32); - } - tmp += sum; - - } - -#endif - // sum up partial sums and write back result tmp = warp_reduce_sum(tmp); diff --git a/ggml-cuda/mmq.cu b/ggml-cuda/mmq.cu index 933d799ce..c0a66d9b6 100644 --- a/ggml-cuda/mmq.cu +++ b/ggml-cuda/mmq.cu @@ -826,11 +826,7 @@ template static __device__ __forceinlin const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd; -#if QK_K == 256 x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = bxi->dm; -#else - x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = {bxi->dm[0], bxi->dm[1]}; -#endif } #pragma unroll @@ -933,9 +929,7 @@ template static __device__ __forceinlin const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd; -#if QK_K == 256 x_dm[i * (WARP_SIZE/QI5_K) + i / QI5_K + kbxd] = bxi->dm; -#endif } #pragma unroll diff --git a/ggml-cuda/vecdotq.cuh b/ggml-cuda/vecdotq.cuh index 86b87fa93..5ebdddcc7 100644 --- a/ggml-cuda/vecdotq.cuh +++ b/ggml-cuda/vecdotq.cuh @@ -712,7 +712,6 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1( static __device__ __forceinline__ float vec_dot_q4_K_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { -#ifndef GGML_QKK_64 const block_q4_K * bq4_K = (const block_q4_K *) vbq; int v[2]; @@ -754,58 +753,11 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1( } return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8); - -#else - -#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics - const block_q4_K * bq4_K = (const block_q4_K *) vbq; - - float sumf_d = 0.0f; - float sumf_m = 0.0f; - - uint16_t aux16[2]; - const uint8_t * s = (const uint8_t *)aux16; - - const uint16_t * a = (const uint16_t *)bq4_K->scales; - aux16[0] = a[0] & 0x0f0f; - aux16[1] = (a[0] >> 4) & 0x0f0f; - - const float dall = bq4_K->dm[0]; - const float dmin = bq4_K->dm[1]; - - const float d8_1 = __low2float(bq8_1[0].ds); - const float d8_2 = __low2float(bq8_1[1].ds); - - const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2)); - const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4); - const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2)); - const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4); - - const int * q4 = (const int *)bq4_K->qs + (iqs/2); - const int v1 = q4[0]; - const int v2 = q4[4]; - - const int dot1 = __dp4a(ui2, v2 & 0x0f0f0f0f, __dp4a(ui1, v1 & 0x0f0f0f0f, 0)); - const int dot2 = __dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, __dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0)); - const int dot3 = __dp4a(0x01010101, ui2, __dp4a(0x01010101, ui1, 0)); - const int dot4 = __dp4a(0x01010101, ui4, __dp4a(0x01010101, ui3, 0)); - - sumf_d += d8_1 * (dot1 * s[0]) + d8_2 * (dot2 * s[1]); - sumf_m += d8_1 * (dot3 * s[2]) + d8_2 * (dot4 * s[3]); - - return dall * sumf_d - dmin * sumf_m; - -#else - NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= MIN_CC_DP4A - -#endif } static __device__ __forceinline__ float vec_dot_q5_K_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { -#ifndef GGML_QKK_64 const block_q5_K * bq5_K = (const block_q5_K *) vbq; int vl[2]; @@ -847,48 +799,6 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1( } return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8); - -#else - -#if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics - const block_q5_K * bq5_K = (const block_q5_K *) vbq; - - const int8_t * s = bq5_K->scales; - - const float d = bq5_K->d; - - const float d8_1 = __low2half(bq8_1[0].ds); - const float d8_2 = __low2half(bq8_1[1].ds); - - const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2)); - const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4); - const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2)); - const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4); - - const int * ql = (const int *)bq5_K->qs + (iqs/2); - const int vl1 = ql[0]; - const int vl2 = ql[4]; - - const int step = 4 * (iqs/2); // 0, 4, 8, 12 - const int im = step/8; // = 0 for iqs = 0, 2, = 1 for iqs = 4, 6 - const int in = step%8; // 0, 4, 0, 4 - const int vh = (*((const int *)(bq5_K->qh + in))) >> im; - - const int v1 = (((vh << 4) & 0x10101010) ^ 0x10101010) | ((vl1 >> 0) & 0x0f0f0f0f); - const int v2 = (((vh << 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 0) & 0x0f0f0f0f); - const int v3 = (((vh >> 0) & 0x10101010) ^ 0x10101010) | ((vl1 >> 4) & 0x0f0f0f0f); - const int v4 = (((vh >> 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 4) & 0x0f0f0f0f); - - const float sumf_d = d8_1 * (__dp4a(ui1, v1, 0) * s[0] + __dp4a(ui2, v2, 0) * s[1]) - + d8_2 * (__dp4a(ui3, v3, 0) * s[2] + __dp4a(ui4, v4, 0) * s[3]); - - return d * sumf_d; - -#else - NO_DEVICE_CODE; -#endif // __CUDA_ARCH__ >= MIN_CC_DP4A - -#endif } static __device__ __forceinline__ float vec_dot_q6_K_q8_1( @@ -919,7 +829,6 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1( static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { -#if QK_K == 256 const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq; #if QR2_XXS == 8 @@ -960,15 +869,11 @@ static __device__ __forceinline__ float vec_dot_iq2_xxs_q8_1( } return d * (sumi1 + sumi2); #endif -#else - NO_DEVICE_CODE; -#endif } static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { #if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics -#if QK_K == 256 const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq; const int ib32 = iqs; @@ -1002,17 +907,12 @@ static __device__ __forceinline__ float vec_dot_iq2_xs_q8_1( GGML_UNUSED(ksigns64); NO_DEVICE_CODE; #endif -#else - GGML_UNUSED(ksigns64); - NO_DEVICE_CODE; -#endif } // TODO static __device__ __forceinline__ float vec_dot_iq2_s_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { #if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics -#if QK_K == 256 const block_iq2_s * bq2 = (const block_iq2_s *) vbq; const int ib32 = iqs; @@ -1048,16 +948,11 @@ static __device__ __forceinline__ float vec_dot_iq2_s_q8_1( GGML_UNUSED(ksigns64); NO_DEVICE_CODE; #endif -#else - GGML_UNUSED(ksigns64); - NO_DEVICE_CODE; -#endif } static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { #if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics -#if QK_K == 256 const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq; const int ib32 = iqs; @@ -1082,16 +977,12 @@ static __device__ __forceinline__ float vec_dot_iq3_xxs_q8_1( #else NO_DEVICE_CODE; #endif -#else - NO_DEVICE_CODE; -#endif } // TODO: don't use lookup table for signs static __device__ __forceinline__ float vec_dot_iq3_s_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { #if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics -#if QK_K == 256 const block_iq3_s * bq2 = (const block_iq3_s *) vbq; const int ib32 = iqs; @@ -1114,14 +1005,10 @@ static __device__ __forceinline__ float vec_dot_iq3_s_q8_1( #else NO_DEVICE_CODE; #endif -#else - NO_DEVICE_CODE; -#endif } static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { -#if QK_K == 256 const block_iq1_s * bq1 = (const block_iq1_s *) vbq; const int ib32 = iqs; @@ -1149,14 +1036,10 @@ static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( const float d = d1q * __low2float (bq8_1[ib32].ds); const float m = d1q * __high2float(bq8_1[ib32].ds); return d * sumi + m * delta; -#else - NO_DEVICE_CODE; -#endif } static __device__ __forceinline__ float vec_dot_iq1_m_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { -#if QK_K == 256 const block_iq1_m * bq1 = (const block_iq1_m *) vbq; const int ib32 = iqs; @@ -1192,9 +1075,6 @@ static __device__ __forceinline__ float vec_dot_iq1_m_q8_1( scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); const float d = (float)scale.f16 * __low2float (bq8_1[ib32].ds); return d * ((sumi[0] + sumf[0]) * (2*((sc[ib32/2] >> 6*(ib32%2)) & 0x7) + 1) + (sumi[1] + sumf[1]) * (2*((sc[ib32/2] >> (6*(ib32%2)+3)) & 0x7) + 1)); -#else - NO_DEVICE_CODE; -#endif } #if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics @@ -1250,9 +1130,7 @@ static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1( static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & iqs) { -#if QK_K == 256 #if __CUDA_ARCH__ >= MIN_CC_DP4A // lowest compute capability for integer intrinsics - const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq; const uint8_t * values = (const uint8_t *)kvalues_iq4nl; @@ -1270,10 +1148,6 @@ static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1( sumi2 = __dp4a(v2, q8[j+4], sumi2); } return d * (sumi1 + sumi2); - -#else - NO_DEVICE_CODE; -#endif #else return vec_dot_iq4_xs_q8_1(vbq, bq8_1, iqs); #endif diff --git a/ggml-metal.m b/ggml-metal.m index 5d5ad20ad..c9e570dbf 100644 --- a/ggml-metal.m +++ b/ggml-metal.m @@ -381,10 +381,6 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) { // dictionary of preprocessor macros NSMutableDictionary * prep = [NSMutableDictionary dictionary]; -#ifdef GGML_QKK_64 - prep[@"GGML_QKK_64"] = @(1); -#endif - MTLCompileOptions* options = [MTLCompileOptions new]; options.preprocessorMacros = prep; @@ -1773,11 +1769,7 @@ static enum ggml_status ggml_metal_graph_compute( [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } else if (src0t == GGML_TYPE_Q3_K) { -#ifdef GGML_QKK_64 - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; -#else [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; -#endif } else if (src0t == GGML_TYPE_Q5_K) { [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, ne11, ne12*ne13) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; @@ -2018,12 +2010,7 @@ static enum ggml_status ggml_metal_graph_compute( { nth0 = 4; nth1 = 16; - #if QK_K == 64 - pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32].pipeline; - #else pipeline = ctx->kernels[GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32].pipeline; - #endif - } break; default: { @@ -2088,11 +2075,7 @@ static enum ggml_status ggml_metal_graph_compute( [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; } else if (src0t == GGML_TYPE_Q3_K) { -#ifdef GGML_QKK_64 - [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 1)/2, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; -#else [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; -#endif } else if (src0t == GGML_TYPE_Q5_K) { [encoder dispatchThreadgroups:MTLSizeMake((ne01 + 3)/4, _ne1, tgz) threadsPerThreadgroup:MTLSizeMake(nth0, nth1, 1)]; diff --git a/ggml-metal.metal b/ggml-metal.metal index c5eb25280..8ff70d7a7 100644 --- a/ggml-metal.metal +++ b/ggml-metal.metal @@ -3386,7 +3386,6 @@ void kernel_mul_mv_q2_K_f32_impl( const int step = sizeof(block_q2_K) * nb; -#if QK_K == 256 const int ix = tiisg/8; // 0...3 const int it = tiisg%8; // 0...7 const int iq = it/4; // 0 or 1 @@ -3438,57 +3437,6 @@ void kernel_mul_mv_q2_K_f32_impl( y4 += 4 * QK_K; } -#else - const int ix = tiisg/2; // 0...15 - const int it = tiisg%2; // 0...1 - - device const float * y4 = y + ix * QK_K + 8 * it; - - for (int ib = ix; ib < nb; ib += 16) { - - float4 sumy = {0.f, 0.f, 0.f, 0.f}; - for (int i = 0; i < 8; ++i) { - yl[i+ 0] = y4[i+ 0]; sumy[0] += yl[i+ 0]; - yl[i+ 8] = y4[i+16]; sumy[1] += yl[i+ 8]; - yl[i+16] = y4[i+32]; sumy[2] += yl[i+16]; - yl[i+24] = y4[i+48]; sumy[3] += yl[i+24]; - } - - device const uint8_t * sc = (device const uint8_t *)x[ib].scales; - device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 4 * it; - device const half * dh = &x[ib].d; - - for (int row = 0; row < N_DST; row++) { - - float4 acc1 = {0.f, 0.f, 0.f, 0.f}; - float4 acc2 = {0.f, 0.f, 0.f, 0.f}; - for (int i = 0; i < 8; i += 2) { - acc1[0] += yl[i+ 0] * (qs[i/2] & 0x0003); - acc2[0] += yl[i+ 1] * (qs[i/2] & 0x0300); - acc1[1] += yl[i+ 8] * (qs[i/2] & 0x000c); - acc2[1] += yl[i+ 9] * (qs[i/2] & 0x0c00); - acc1[2] += yl[i+16] * (qs[i/2] & 0x0030); - acc2[2] += yl[i+17] * (qs[i/2] & 0x3000); - acc1[3] += yl[i+24] * (qs[i/2] & 0x00c0); - acc2[3] += yl[i+25] * (qs[i/2] & 0xc000); - } - - float dall = dh[0]; - float dmin = dh[1]; - sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc2[0]) * (sc[0] & 0xF) * 1.f/ 1.f + - (acc1[1] + 1.f/256.f * acc2[1]) * (sc[1] & 0xF) * 1.f/ 4.f + - (acc1[2] + 1.f/256.f * acc2[2]) * (sc[2] & 0xF) * 1.f/16.f + - (acc1[3] + 1.f/256.f * acc2[3]) * (sc[3] & 0xF) * 1.f/64.f) - - dmin * (sumy[0] * (sc[0] >> 4) + sumy[1] * (sc[1] >> 4) + sumy[2] * (sc[2] >> 4) + sumy[3] * (sc[3] >> 4)); - - qs += step/2; - sc += step; - dh += step/2; - } - - y4 += 16 * QK_K; - } -#endif for (int row = 0; row < N_DST; ++row) { all_sum = simd_sum(sumf[row]); @@ -3526,7 +3474,6 @@ kernel void kernel_mul_mv_q2_K_f32( kernel_mul_mv_q2_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg); } -#if QK_K == 256 void kernel_mul_mv_q3_K_f32_impl( device const void * src0, device const float * src1, @@ -3685,84 +3632,6 @@ void kernel_mul_mv_q3_K_f32_impl( } } } -#else -void kernel_mul_mv_q3_K_f32_impl( - device const void * src0, - device const float * src1, - device float * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant int64_t & ne10, - constant int64_t & ne12, - constant int64_t & ne0, - constant int64_t & ne1, - constant uint & r2, - constant uint & r3, - threadgroup int8_t * shared_values [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - uint tiisg[[thread_index_in_simdgroup]], - uint sgitg[[simdgroup_index_in_threadgroup]]) { - - const int nb = ne00/QK_K; - - const int64_t r0 = tgpig.x; - const int64_t r1 = tgpig.y; - const int64_t im = tgpig.z; - - const int row = 2 * r0 + sgitg; - - const uint i12 = im%ne12; - const uint i13 = im/ne12; - - const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); - - device const block_q3_K * x = (device const block_q3_K *) src0 + row*nb + offset0; - device const float * yy = (device const float *) src1 + r1*ne10 + im*ne00*ne1; - - const int ix = tiisg/4; - const int il = 4 * (tiisg%4);// 0, 4, 8, 12 - const int iq = il/8; // 0, 0, 1, 1 - const int in = il%8; // 0, 4, 0, 4 - - float2 sum = {0.f, 0.f}; - - for (int i = ix; i < nb; i += 8) { - - const float d_all = (float)(x[i].d); - - device const uint16_t * q = (device const uint16_t *)(x[i].qs + il); - device const uint16_t * h = (device const uint16_t *)(x[i].hmask + in); - device const uint16_t * s = (device const uint16_t *)(x[i].scales); - device const float * y = yy + i * QK_K + il; - - const float d1 = d_all * ((int32_t)(s[0] & 0x000F) - 8); - const float d2 = d_all * ((int32_t)(s[0] & 0x00F0) - 128) * 1.f/64.f; - const float d3 = d_all * ((int32_t)(s[0] & 0x0F00) - 2048) * 1.f/4096.f; - const float d4 = d_all * ((int32_t)(s[0] & 0xF000) - 32768) * 1.f/262144.f; - - for (int l = 0; l < 4; l += 2) { - const uint16_t hm = h[l/2] >> iq; - sum[0] += y[l+ 0] * d1 * ((int32_t)(q[l/2] & 0x0003) - ((hm & 0x0001) ? 0 : 4)) - + y[l+16] * d2 * ((int32_t)(q[l/2] & 0x000c) - ((hm & 0x0004) ? 0 : 16)) - + y[l+32] * d3 * ((int32_t)(q[l/2] & 0x0030) - ((hm & 0x0010) ? 0 : 64)) - + y[l+48] * d4 * ((int32_t)(q[l/2] & 0x00c0) - ((hm & 0x0040) ? 0 : 256)); - sum[1] += y[l+ 1] * d1 * ((int32_t)(q[l/2] & 0x0300) - ((hm & 0x0100) ? 0 : 1024)) - + y[l+17] * d2 * ((int32_t)(q[l/2] & 0x0c00) - ((hm & 0x0400) ? 0 : 4096)) - + y[l+33] * d3 * ((int32_t)(q[l/2] & 0x3000) - ((hm & 0x1000) ? 0 : 16384)) - + y[l+49] * d4 * ((int32_t)(q[l/2] & 0xc000) - ((hm & 0x4000) ? 0 : 65536)); - } - - } - const float sumf = sum[0] + sum[1] * 1.f/256.f; - - const float tot = simd_sum(sumf); - if (tiisg == 0) { - dst[r1*ne0 + im*ne0*ne1 + row] = tot; - } - -} -#endif [[host_name("kernel_mul_mv_q3_K_f32")]] kernel void kernel_mul_mv_q3_K_f32( @@ -3792,7 +3661,6 @@ kernel void kernel_mul_mv_q3_K_f32( kernel_mul_mv_q3_K_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, nullptr, tgpig, tiisg, sgitg); } -#if QK_K == 256 void kernel_mul_mv_q4_K_f32_impl( device const void * src0, device const float * src1, @@ -3906,103 +3774,6 @@ void kernel_mul_mv_q4_K_f32_impl( } } } -#else -void kernel_mul_mv_q4_K_f32_impl( - device const void * src0, - device const float * src1, - device float * dst, - constant int64_t & ne00, - constant int64_t & ne01, - constant int64_t & ne02, - constant int64_t & ne10, - constant int64_t & ne12, - constant int64_t & ne0, - constant int64_t & ne1, - constant uint & r2, - constant uint & r3, - threadgroup int8_t * shared_values [[threadgroup(0)]], - uint3 tgpig[[threadgroup_position_in_grid]], - uint tiisg[[thread_index_in_simdgroup]], - uint sgitg[[simdgroup_index_in_threadgroup]]) { - - const int ix = tiisg/4; // 0...7 - const int it = tiisg%4; // 0...3 - - const int nb = ne00/QK_K; - const int r0 = tgpig.x; - const int r1 = tgpig.y; - const int im = tgpig.z; - const int first_row = r0 * N_DST; - const int ib_row = first_row * nb; - - const uint i12 = im%ne12; - const uint i13 = im/ne12; - - const uint offset0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02); - - device const block_q4_K * x = (device const block_q4_K *) src0 + ib_row + offset0; - device const float * y = (device const float *) src1 + r1*ne10 + im*ne00*ne1; - - float yl[8]; - float yh[8]; - float sumf[N_DST]={0.f}, all_sum; - - const int step = sizeof(block_q4_K) * nb / 2; - - device const float * y4 = y + ix * QK_K + 8 * it; - - uint16_t sc16[4]; - - for (int ib = ix; ib < nb; ib += 8) { - - float2 sumy = {0.f, 0.f}; - for (int i = 0; i < 8; ++i) { - yl[i] = y4[i+ 0]; sumy[0] += yl[i]; - yh[i] = y4[i+32]; sumy[1] += yh[i]; - } - - device const uint16_t * sc = (device const uint16_t *)x[ib].scales; - device const uint16_t * qs = (device const uint16_t *)x[ib].qs + 4 * it; - device const half * dh = x[ib].d; - - for (int row = 0; row < N_DST; row++) { - - sc16[0] = sc[0] & 0x000f; - sc16[1] = sc[0] & 0x0f00; - sc16[2] = sc[0] & 0x00f0; - sc16[3] = sc[0] & 0xf000; - - float2 acc1 = {0.f, 0.f}; - float2 acc2 = {0.f, 0.f}; - for (int i = 0; i < 8; i += 2) { - acc1[0] += yl[i+0] * (qs[i/2] & 0x000F); - acc1[1] += yl[i+1] * (qs[i/2] & 0x0F00); - acc2[0] += yh[i+0] * (qs[i/2] & 0x00F0); - acc2[1] += yh[i+1] * (qs[i/2] & 0xF000); - } - - float dall = dh[0]; - float dmin = dh[1]; - sumf[row] += dall * ((acc1[0] + 1.f/256.f * acc1[1]) * sc16[0] + - (acc2[0] + 1.f/256.f * acc2[1]) * sc16[1] * 1.f/4096.f) - - dmin * 1.f/16.f * (sumy[0] * sc16[2] + sumy[1] * sc16[3] * 1.f/256.f); - - qs += step; - sc += step; - dh += step; - } - - y4 += 8 * QK_K; - } - - for (int row = 0; row < N_DST; ++row) { - all_sum = simd_sum(sumf[row]); - if (tiisg == 0) { - dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum; - } - } -} -#endif [[host_name("kernel_mul_mv_q4_K_f32")]] kernel void kernel_mul_mv_q4_K_f32( @@ -4070,8 +3841,6 @@ void kernel_mul_mv_q5_K_f32_impl( const int step = sizeof(block_q5_K) * nb; -#if QK_K == 256 -# float yl[16], yh[16]; const uint16_t kmask1 = 0x3f3f; @@ -4154,54 +3923,6 @@ void kernel_mul_mv_q5_K_f32_impl( y1 += 4 * QK_K; } -#else - float yl[8], yh[8]; - - const int il = 4 * (tiisg/8); // 0, 4, 8, 12 - const int ix = tiisg%8; - const int iq = il/8; // 0, 0, 1, 1 - const int in = il%8; // 0, 4, 0, 4 - - device const float * y = yy + ix*QK_K + il; - - for (int i = ix; i < nb; i += 8) { - - for (int l = 0; l < 4; ++l) { - yl[l+0] = y[l+ 0]; - yl[l+4] = y[l+16]; - yh[l+0] = y[l+32]; - yh[l+4] = y[l+48]; - } - - device const half * dh = &x[i].d; - device const uint8_t * q = x[i].qs + il; - device const uint8_t * h = x[i].qh + in; - device const int8_t * s = x[i].scales; - - for (int row = 0; row < 2; ++row) { - - const float d = dh[0]; - - float2 acc = {0.f, 0.f}; - for (int l = 0; l < 4; ++l) { - const uint8_t hl = h[l] >> iq; - acc[0] += yl[l+0] * s[0] * ((int16_t)(q[l+ 0] & 0x0F) - (hl & 0x01 ? 0 : 16)) - + yl[l+4] * s[1] * ((int16_t)(q[l+16] & 0x0F) - (hl & 0x04 ? 0 : 16)); - acc[1] += yh[l+0] * s[2] * ((int16_t)(q[l+ 0] & 0xF0) - (hl & 0x10 ? 0 : 256)) - + yh[l+4] * s[3] * ((int16_t)(q[l+16] & 0xF0) - (hl & 0x40 ? 0 : 256)); - } - sumf[row] += d * (acc[0] + 1.f/16.f * acc[1]); - - q += step; - h += step; - s += step; - dh += step/2; - - } - - y += 8 * QK_K; - } -#endif for (int row = 0; row < 2; ++row) { const float tot = simd_sum(sumf[row]); @@ -4280,7 +4001,6 @@ void kernel_mul_mv_q6_K_f32_impl( float sumf = 0; -#if QK_K == 256 const int tid = tiisg/2; const int ix = tiisg%2; const int ip = tid/8; // 0 or 1 @@ -4316,30 +4036,6 @@ void kernel_mul_mv_q6_K_f32_impl( } -#else - const int ix = tiisg/4; - const int il = 4*(tiisg%4); - - for (int i = ix; i < nb; i += 8) { - device const float * y = yy + i * QK_K + il; - device const uint8_t * ql = x[i].ql + il; - device const uint8_t * qh = x[i].qh + il; - device const int8_t * s = x[i].scales; - - const float d = x[i].d; - - float4 sums = {0.f, 0.f, 0.f, 0.f}; - for (int l = 0; l < 4; ++l) { - sums[0] += y[l+ 0] * ((int8_t)((ql[l+ 0] & 0xF) | ((qh[l] & kmask1) << 4)) - 32); - sums[1] += y[l+16] * ((int8_t)((ql[l+16] & 0xF) | ((qh[l] & kmask2) << 2)) - 32); - sums[2] += y[l+32] * ((int8_t)((ql[l+ 0] >> 4) | ((qh[l] & kmask3) >> 0)) - 32); - sums[3] += y[l+48] * ((int8_t)((ql[l+16] >> 4) | ((qh[l] & kmask4) >> 2)) - 32); - } - sumf += d * (sums[0] * s[0] + sums[1] * s[1] + sums[2] * s[2] + sums[3] * s[3]); - } - -#endif - const float tot = simd_sum(sumf); if (tiisg == 0) { dst[r1*ne0 + im*ne0*ne1 + row] = tot; @@ -5173,9 +4869,7 @@ void kernel_mul_mv_iq1_m_f32_impl( device const float * y4 = y + 32 * ix; -#if QK_K != 64 iq1m_scale_t scale; -#endif for (int ib32 = ix; ib32 < nb32; ib32 += 32) { @@ -5196,10 +4890,7 @@ void kernel_mul_mv_iq1_m_f32_impl( device const uint16_t * sc = (device const uint16_t *)xr->scales; for (int row = 0; row < N_DST; row++) { - -#if QK_K != 64 scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); -#endif constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); constant uint8_t * grid2 = (constant uint8_t *)(iq1s_grid_gpu + (qs[1] | ((qh[0] << 4) & 0x700))); @@ -5215,14 +4906,9 @@ void kernel_mul_mv_iq1_m_f32_impl( } const float delta1 = sumy[0] * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[1] * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); const float delta2 = sumy[2] * (qh[1] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA) + sumy[3] * (qh[1] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); -#if QK_K == 64 - const float d = (float) *((device const half *)(sc - 1)); - sumf[row] += d * ((sum[0] + delta1) * (2*((sc[0] >> (8*(ib%2)+0)) & 0xf) + 1) + - (sum[1] + delta2) * (2*((sc[0] >> (8*(ib%2)+4)) & 0xf) + 1)); -#else + sumf[row] += (float)scale.f16 * ((sum[0] + delta1) * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 7) + 1) + (sum[1] + delta2) * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 7) + 1)); -#endif sc += nb*sizeof(block_iq1_m)/2; qs += nb*sizeof(block_iq1_m); @@ -5334,7 +5020,6 @@ void kernel_mul_mv_iq4_nl_f32_impl( } } -#if QK_K != 64 void kernel_mul_mv_iq4_xs_f32_impl( device const void * src0, device const float * src1, @@ -5429,7 +5114,6 @@ void kernel_mul_mv_iq4_xs_f32_impl( } } } -#endif [[host_name("kernel_mul_mv_iq1_s_f32")]] kernel void kernel_mul_mv_iq1_s_f32( @@ -5542,11 +5226,7 @@ kernel void kernel_mul_mv_iq4_xs_f32( uint tiisg[[thread_index_in_simdgroup]], uint sgitg[[simdgroup_index_in_threadgroup]]) { -#if QK_K == 64 - kernel_mul_mv_iq4_nl_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); -#else kernel_mul_mv_iq4_xs_f32_impl(src0, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3, shared_values, tgpig, tiisg, sgitg); -#endif } //============================= templates and their specializations ============================= @@ -5672,10 +5352,9 @@ void dequantize_q2_K(device const block_q2_K *xb, short il, thread type4x4 & reg float dl, ml; uint8_t sc = xb->scales[il]; -#if QK_K == 256 q = q + 32*(il/8) + 16*(il&1); il = (il/2)%4; -#endif + half coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h); uchar mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3); dl = d * (sc & 0xF) * coef, ml = min * (sc >> 4); @@ -5691,7 +5370,6 @@ void dequantize_q3_K(device const block_q3_K *xb, short il, thread type4x4 & reg device const uint8_t * h = (device const uint8_t *)xb->hmask; device const int8_t * scales = (device const int8_t *)xb->scales; -#if QK_K == 256 q = q + 32 * (il/8) + 16 * (il&1); h = h + 16 * (il&1); uint8_t m = 1 << (il/2); @@ -5712,17 +5390,6 @@ void dequantize_q3_K(device const block_q3_K *xb, short il, thread type4x4 & reg for (int i = 0; i < 16; ++i) { reg[i/4][i%4] = dl * (q[i] & mask) - (h[i] & m ? 0 : ml); } -#else - float kcoef = il&1 ? 1.f/16.f : 1.f; - uint16_t kmask = il&1 ? 0xF0 : 0x0F; - float dl = d_all * ((scales[il/2] & kmask) * kcoef - 8); - float coef = il>1 ? (il>2 ? 1/64.h : 1/16.h) : (il>0 ? 1/4.h : 1.h); - uint8_t mask = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3); - uint8_t m = 1<<(il*2); - for (int i = 0; i < 16; ++i) { - reg[i/4][i%4] = coef * dl * ((q[i] & mask) - ((h[i%8] & (m * (1 + i/8))) ? 0 : 4.f/coef)); - } -#endif } static inline uchar2 get_scale_min_k4_just2(int j, int k, device const uchar * q) { @@ -5734,7 +5401,6 @@ template void dequantize_q4_K(device const block_q4_K *xb, short il, thread type4x4 & reg) { device const uchar * q = xb->qs; -#if QK_K == 256 short is = (il/4) * 2; q = q + (il/4) * 32 + 16 * (il&1); il = il & 3; @@ -5743,16 +5409,7 @@ void dequantize_q4_K(device const block_q4_K *xb, short il, thread type4x4 & reg const float min = xb->dmin; const float dl = d * sc[0]; const float ml = min * sc[1]; -#else - (void) get_scale_min_k4_just2; - q = q + 16 * (il&1); - device const uint8_t * s = xb->scales; - device const half2 * dh = (device const half2 *)xb->d; - const float2 d = (float2)dh[0]; - const float dl = il<2 ? d[0] * (s[0]&0xF) : d[0] * (s[1]&0xF)/16.h; - const float ml = il<2 ? d[1] * (s[0]>>4) : d[1] * (s[1]>>4); -#endif const ushort mask = il<2 ? 0x0F : 0xF0; for (int i = 0; i < 16; ++i) { reg[i/4][i%4] = dl * (q[i] & mask) - ml; @@ -5764,7 +5421,6 @@ void dequantize_q5_K(device const block_q5_K *xb, short il, thread type4x4 & reg device const uint8_t * q = xb->qs; device const uint8_t * qh = xb->qh; -#if QK_K == 256 short is = (il/4) * 2; q = q + 32 * (il/4) + 16 * (il&1); qh = qh + 16 * (il&1); @@ -5781,17 +5437,6 @@ void dequantize_q5_K(device const block_q5_K *xb, short il, thread type4x4 & reg for (int i = 0; i < 16; ++i) { reg[i/4][i%4] = dl * ((q[i] & mask) + (qh[i] & ul ? qh_val : 0)) - ml; } -#else - q = q + 16 * (il&1); - device const int8_t * s = xb->scales; - const float dl = xb->d * s[il]; - uint8_t m = 1<<(il*2); - const float coef = il<2 ? 1.f : 1.f/16.f; - const ushort mask = il<2 ? 0x0F : 0xF0; - for (int i = 0; i < 16; ++i) { - reg[i/4][i%4] = coef * dl * ((q[i] & mask) - (qh[i%8] & (m*(1+i/8)) ? 0.f : 16.f/coef)); - } -#endif } template @@ -5801,15 +5446,11 @@ void dequantize_q6_K(device const block_q6_K *xb, short il, thread type4x4 & reg device const uint8_t * qh = (device const uint8_t *)xb->qh; device const int8_t * scales = (device const int8_t *)xb->scales; -#if QK_K == 256 ql = ql + 64*(il/8) + 32*((il/2)&1) + 16*(il&1); qh = qh + 32*(il/8) + 16*(il&1); float sc = scales[(il%2) + 2 * ((il/2))]; il = (il/2) & 3; -#else - ql = ql + 16 * (il&1); - float sc = scales[il]; -#endif + const uint16_t kmask1 = il>1 ? (il>2 ? 192 : 48) : (il>0 ? 12 : 3); const uint16_t kmask2 = il>1 ? 0xF0 : 0x0F; const float coef = il>1 ? 1.f/16.f : 1.f; @@ -5966,20 +5607,15 @@ void dequantize_iq1_m(device const block_iq1_m * xb, short il, thread type4x4 & const int ib32 = il/2; il = il%2; device const uint16_t * sc = (device const uint16_t *)xb->scales; -#if QK_K == 64 - const float d = xb->d; -#else + iq1m_scale_t scale; scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); const float d = scale.f16; -#endif + device const uint8_t * qs = xb->qs + 4*ib32 + 2*il; device const uint8_t * qh = xb->qh + 2*ib32 + il; -#if QK_K == 64 - const float dl = d * (2*((sc[ib32/2] >> (8*(ib32%2)+4*il)) & 0xf) + 1); -#else + const float dl = d * (2*((sc[ib32/2] >> (6*(ib32%2)+3*il)) & 7) + 1); -#endif const float ml1 = dl * (qh[0] & 0x08 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); const float ml2 = dl * (qh[0] & 0x80 ? -1 - IQ1M_DELTA : -1 + IQ1M_DELTA); constant uint8_t * grid1 = (constant uint8_t *)(iq1s_grid_gpu + (qs[0] | ((qh[0] << 8) & 0x700))); @@ -6009,9 +5645,6 @@ void dequantize_iq4_nl(device const block_iq4_nl * xb, short il, thread type4x4 template void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 & reg) { -#if QK_K == 64 - dequantize_iq4_nl(xb, il, reg); -#else // il is 0...15 for QK_K = 256 => index of block of 32 is il/2 const int ib32 = il/2; il = il%2; @@ -6028,7 +5661,6 @@ void dequantize_iq4_xs(device const block_iq4_xs * xb, short il, thread type4x4 reg[i][2] = d * kvalues_iq4nl_f[q8[2]]; reg[i][3] = d * kvalues_iq4nl_f[q8[3]]; } -#endif } template @@ -6533,11 +6165,7 @@ kernel void kernel_mul_mm_id( sgitg); } -#if QK_K == 256 #define QK_NL 16 -#else -#define QK_NL 4 -#endif // // get rows @@ -6577,11 +6205,7 @@ template [[host_name("kernel_get_rows_iq2_s")]] kernel get_rows_t kernel_get_r template [[host_name("kernel_get_rows_iq1_s")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq1_m")]] kernel get_rows_t kernel_get_rows; template [[host_name("kernel_get_rows_iq4_nl")]] kernel get_rows_t kernel_get_rows; -#if QK_K == 64 -template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_t kernel_get_rows; -#else template [[host_name("kernel_get_rows_iq4_xs")]] kernel get_rows_t kernel_get_rows; -#endif // // matrix-matrix multiplication @@ -6609,11 +6233,7 @@ template [[host_name("kernel_mul_mm_iq2_s_f32")]] kernel mat_mm_t kernel_mul_m template [[host_name("kernel_mul_mm_iq1_s_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq1_m_f32")]] kernel mat_mm_t kernel_mul_mm; template [[host_name("kernel_mul_mm_iq4_nl_f32")]] kernel mat_mm_t kernel_mul_mm; -#if QK_K == 64 -template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mat_mm_t kernel_mul_mm; -#else template [[host_name("kernel_mul_mm_iq4_xs_f32")]] kernel mat_mm_t kernel_mul_mm; -#endif // // indirect matrix-matrix multiplication @@ -6641,11 +6261,7 @@ template [[host_name("kernel_mul_mm_id_iq2_s_f32")]] kernel mat_mm_id_t kernel template [[host_name("kernel_mul_mm_id_iq1_s_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq1_m_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; template [[host_name("kernel_mul_mm_id_iq4_nl_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -#if QK_K == 64 -template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -#else template [[host_name("kernel_mul_mm_id_iq4_xs_f32")]] kernel mat_mm_id_t kernel_mul_mm_id; -#endif // // matrix-vector multiplication @@ -6854,7 +6470,5 @@ template [[host_name("kernel_mul_mv_id_iq3_xxs_f32")]] kernel kernel_mul_mv_id_t template [[host_name("kernel_mul_mv_id_iq3_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>; template [[host_name("kernel_mul_mv_id_iq2_s_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>; template [[host_name("kernel_mul_mv_id_iq4_nl_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>; -#if QK_K != 64 template [[host_name("kernel_mul_mv_id_iq4_xs_f32")]] kernel kernel_mul_mv_id_t kernel_mul_mv_id>; -#endif diff --git a/ggml-opencl.cpp b/ggml-opencl.cpp index 922f24837..e28566a7b 100644 --- a/ggml-opencl.cpp +++ b/ggml-opencl.cpp @@ -1,4 +1,4 @@ -#include "ggml.h" +#include "ggml.h" #include "ggml-opencl.h" #include "ggml-backend-impl.h" diff --git a/ggml-quants.c b/ggml-quants.c index ed40ca74a..88f58a339 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -1888,7 +1888,6 @@ static float make_qkx2_quants(int n, int nmax, const float * restrict x, const f return scale; } -#if QK_K == 256 static inline void get_scale_min_k4(int j, const uint8_t * restrict q, uint8_t * restrict d, uint8_t * restrict m) { if (j < 4) { *d = q[j] & 63; *m = q[j + 4] & 63; @@ -1897,7 +1896,6 @@ static inline void get_scale_min_k4(int j, const uint8_t * restrict q, uint8_t * *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); } } -#endif //========================- 2-bit (de)-quantization @@ -1961,20 +1959,13 @@ void quantize_row_q2_K_reference(const float * restrict x, block_q2_K * restrict } } -#if QK_K == 256 for (int j = 0; j < QK_K; j += 128) { for (int l = 0; l < 32; ++l) { y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); } } -#else - for (int l = 0; l < 16; ++l) { - y[i].qs[l] = L[l] | (L[l + 16] << 2) | (L[l + 32] << 4) | (L[l + 48] << 6); - } -#endif x += QK_K; - } } @@ -1989,7 +1980,6 @@ void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int6 const uint8_t * q = x[i].qs; -#if QK_K == 256 int is = 0; float dl, ml; for (int n = 0; n < QK_K; n += 128) { @@ -2008,19 +1998,6 @@ void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int6 } q += 32; } -#else - float dl1 = d * (x[i].scales[0] & 0xF), ml1 = min * (x[i].scales[0] >> 4); - float dl2 = d * (x[i].scales[1] & 0xF), ml2 = min * (x[i].scales[1] >> 4); - float dl3 = d * (x[i].scales[2] & 0xF), ml3 = min * (x[i].scales[2] >> 4); - float dl4 = d * (x[i].scales[3] & 0xF), ml4 = min * (x[i].scales[3] >> 4); - for (int l = 0; l < 16; ++l) { - y[l+ 0] = dl1 * ((int8_t)((q[l] >> 0) & 3)) - ml1; - y[l+16] = dl2 * ((int8_t)((q[l] >> 2) & 3)) - ml2; - y[l+32] = dl3 * ((int8_t)((q[l] >> 4) & 3)) - ml3; - y[l+48] = dl4 * ((int8_t)((q[l] >> 6) & 3)) - ml4; - } - y += QK_K; -#endif } } @@ -2211,36 +2188,9 @@ static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restri } float dm, mm; -#if QK_K == 64 - float max_scale = 0, max_min = 0; - for (int j = 0; j < QK_K/16; ++j) { - max_scale = MAX(max_scale, scales[j]); - max_min = MAX(max_min, mins[j]); - } - dm = max_scale/15; - mm = max_min/15; - if (max_scale) { - float id = 1/dm; - for (int j = 0; j < QK_K/16; ++j) { - int l = nearest_int(id*scales[j]); - Ls[j] = MAX(0, MIN(15, l)); - } - } else { - memset(Ls, 0, QK_K/16); - } - if (max_min) { - float id = 1/mm; - for (int j = 0; j < QK_K/16; ++j) { - int l = nearest_int(id*mins[j]); - Lm[j] = MAX(0, MIN(15, l)); - } - } else { - memset(Lm, 0, QK_K/16); - } -#else dm = make_qp_quants(QK_K/16, 15, scales, Ls, sw); mm = make_qp_quants(QK_K/16, 15, mins, Lm, sw); -#endif + y[i].d = GGML_FP32_TO_FP16(dm); y[i].dmin = GGML_FP32_TO_FP16(mm); dm = GGML_FP16_TO_FP32(y[i].d); @@ -2263,20 +2213,13 @@ static void quantize_row_q2_K_impl(const float * restrict x, block_q2_K * restri } } -#if QK_K == 256 for (int j = 0; j < QK_K; j += 128) { for (int l = 0; l < 32; ++l) { y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); } } -#else - for (int l = 0; l < 16; ++l) { - y[i].qs[l] = L[l] | (L[l + 16] << 2) | (L[l + 32] << 4) | (L[l + 48] << 6); - } -#endif x += QK_K; - } } @@ -2317,7 +2260,6 @@ void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict } } -#if QK_K == 256 memset(y[i].scales, 0, 12); if (max_scale) { float iscale = -32.f/max_scale; @@ -2351,36 +2293,6 @@ void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict L[16*j + ii] = l + 4; } } -#else - if (max_scale) { - float iscale = -8.f/max_scale; - for (int j = 0; j < QK_K/16; j+=2) { - int l1 = nearest_int(iscale*scales[j]); - l1 = 8 + MAX(-8, MIN(7, l1)); - int l2 = nearest_int(iscale*scales[j+1]); - l2 = 8 + MAX(-8, MIN(7, l2)); - y[i].scales[j/2] = l1 | (l2 << 4); - } - y[i].d = GGML_FP32_TO_FP16(1/iscale); - } else { - for (int j = 0; j < QK_K/16; j+=2) { - y[i].scales[j/2] = 0; - } - y[i].d = GGML_FP32_TO_FP16(0.f); - } - for (int j = 0; j < QK_K/16; ++j) { - int s = j%2 == 0 ? y[i].scales[j/2] & 0xF : y[i].scales[j/2] >> 4; - float d = GGML_FP16_TO_FP32(y[i].d) * (s - 8); - if (!d) { - continue; - } - for (int ii = 0; ii < 16; ++ii) { - int l = nearest_int(x[16*j + ii]/d); - l = MAX(-4, MIN(3, l)); - L[16*j + ii] = l + 4; - } - } -#endif memset(y[i].hmask, 0, QK_K/8); // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc. @@ -2395,23 +2307,16 @@ void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict m = 0; hm <<= 1; } } -#if QK_K == 256 for (int j = 0; j < QK_K; j += 128) { for (int l = 0; l < 32; ++l) { y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); } } -#else - for (int l = 0; l < 16; ++l) { - y[i].qs[l] = L[l] | (L[l + 16] << 2) | (L[l + 32] << 4) | (L[l + 48] << 6); - } -#endif x += QK_K; } } -#if QK_K == 256 void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int64_t k) { assert(k % QK_K == 0); const int nb = k / QK_K; @@ -2461,49 +2366,12 @@ void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int6 } } -#else -void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int64_t k) { - assert(k % QK_K == 0); - assert(QK_K == 64); - const int nb = k / QK_K; - - for (int i = 0; i < nb; i++) { - - const float d_all = GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q = x[i].qs; - const uint8_t * restrict hm = x[i].hmask; - - const float d1 = d_all * ((x[i].scales[0] & 0xF) - 8); - const float d2 = d_all * ((x[i].scales[0] >> 4) - 8); - const float d3 = d_all * ((x[i].scales[1] & 0xF) - 8); - const float d4 = d_all * ((x[i].scales[1] >> 4) - 8); - - for (int l=0; l<8; ++l) { - uint8_t h = hm[l]; - y[l+ 0] = d1 * ((int8_t)((q[l+0] >> 0) & 3) - ((h & 0x01) ? 0 : 4)); - y[l+ 8] = d1 * ((int8_t)((q[l+8] >> 0) & 3) - ((h & 0x02) ? 0 : 4)); - y[l+16] = d2 * ((int8_t)((q[l+0] >> 2) & 3) - ((h & 0x04) ? 0 : 4)); - y[l+24] = d2 * ((int8_t)((q[l+8] >> 2) & 3) - ((h & 0x08) ? 0 : 4)); - y[l+32] = d3 * ((int8_t)((q[l+0] >> 4) & 3) - ((h & 0x10) ? 0 : 4)); - y[l+40] = d3 * ((int8_t)((q[l+8] >> 4) & 3) - ((h & 0x20) ? 0 : 4)); - y[l+48] = d4 * ((int8_t)((q[l+0] >> 6) & 3) - ((h & 0x40) ? 0 : 4)); - y[l+56] = d4 * ((int8_t)((q[l+8] >> 6) & 3) - ((h & 0x80) ? 0 : 4)); - } - y += QK_K; - } -} -#endif void quantize_row_q3_K(const float * restrict x, void * restrict vy, int64_t k) { quantize_row_q3_K_reference(x, vy, k); } static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restrict y, int64_t n_per_row, const float * restrict quant_weights) { -#if QK_K != 256 - (void)quant_weights; - quantize_row_q3_K_reference(x, y, n_per_row); -#else assert(n_per_row % QK_K == 0); const int nb = n_per_row / QK_K; @@ -2585,7 +2453,6 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restri x += QK_K; } -#endif } size_t quantize_q3_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { @@ -2617,7 +2484,6 @@ void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict float scales[QK_K/32]; for (int i = 0; i < nb; i++) { - float max_scale = 0; // as we are deducting the min, scales are always positive float max_min = 0; for (int j = 0; j < QK_K/32; ++j) { @@ -2637,7 +2503,6 @@ void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict } } -#if QK_K == 256 float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f; float inv_min = max_min > 0 ? 63.f/max_min : 0.f; for (int j = 0; j < QK_K/32; ++j) { @@ -2669,39 +2534,7 @@ void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict L[32*j + ii] = l; } } -#else - const float s_factor = 15.f; - float inv_scale = max_scale > 0 ? s_factor/max_scale : 0.f; - float inv_min = max_min > 0 ? s_factor/max_min : 0.f; - int d1 = nearest_int(inv_scale*scales[0]); - int m1 = nearest_int(inv_min*mins[0]); - int d2 = nearest_int(inv_scale*scales[1]); - int m2 = nearest_int(inv_min*mins[1]); - y[i].scales[0] = d1 | (m1 << 4); - y[i].scales[1] = d2 | (m2 << 4); - y[i].d[0] = GGML_FP32_TO_FP16(max_scale/s_factor); - y[i].d[1] = GGML_FP32_TO_FP16(max_min/s_factor); - float sumlx = 0; - int suml2 = 0; - for (int j = 0; j < QK_K/32; ++j) { - const uint8_t sd = y[i].scales[j] & 0xF; - const uint8_t sm = y[i].scales[j] >> 4; - const float d = GGML_FP16_TO_FP32(y[i].d[0]) * sd; - if (!d) continue; - const float m = GGML_FP16_TO_FP32(y[i].d[1]) * sm; - for (int ii = 0; ii < 32; ++ii) { - int l = nearest_int((x[32*j + ii] + m)/d); - l = MAX(0, MIN(15, l)); - L[32*j + ii] = l; - sumlx += (x[32*j + ii] + m)*l*sd; - suml2 += l*l*sd*sd; - } - } - if (suml2) { - y[i].d[0] = GGML_FP32_TO_FP16(sumlx/suml2); - } -#endif uint8_t * q = y[i].qs; for (int j = 0; j < QK_K; j += 64) { for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4); @@ -2709,7 +2542,6 @@ void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict } x += QK_K; - } } @@ -2718,11 +2550,8 @@ void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int6 const int nb = k / QK_K; for (int i = 0; i < nb; i++) { - const uint8_t * q = x[i].qs; -#if QK_K == 256 - const float d = GGML_FP16_TO_FP32(x[i].d); const float min = GGML_FP16_TO_FP32(x[i].dmin); @@ -2737,18 +2566,6 @@ void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int6 for (int l = 0; l < 32; ++l) *y++ = d2 * (q[l] >> 4) - m2; q += 32; is += 2; } -#else - const float dall = GGML_FP16_TO_FP32(x[i].d[0]); - const float mall = GGML_FP16_TO_FP32(x[i].d[1]); - const float d1 = dall * (x[i].scales[0] & 0xF), m1 = mall * (x[i].scales[0] >> 4); - const float d2 = dall * (x[i].scales[1] & 0xF), m2 = mall * (x[i].scales[1] >> 4); - for (int l = 0; l < 32; ++l) { - y[l+ 0] = d1 * (q[l] & 0xF) - m1; - y[l+32] = d2 * (q[l] >> 4) - m2; - } - y += QK_K; -#endif - } } @@ -2759,10 +2576,6 @@ void quantize_row_q4_K(const float * restrict x, void * restrict vy, int64_t k) } static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restrict y, int64_t n_per_row, const float * quant_weights) { -#if QK_K != 256 - (void)quant_weights; - quantize_row_q4_K_reference(x, y, n_per_row); -#else assert(n_per_row % QK_K == 0); const int64_t nb = n_per_row / QK_K; @@ -2833,7 +2646,6 @@ static void quantize_row_q4_K_impl(const float * restrict x, block_q4_K * restri x += QK_K; } -#endif } size_t quantize_q4_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { @@ -2858,21 +2670,13 @@ void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict assert(k % QK_K == 0); const int64_t nb = k / QK_K; -#if QK_K == 256 uint8_t L[QK_K]; float mins[QK_K/32]; float scales[QK_K/32]; float weights[32]; uint8_t Laux[32]; -#else - int8_t L[QK_K]; - float scales[QK_K/16]; -#endif for (int i = 0; i < nb; i++) { - -#if QK_K == 256 - float max_scale = 0; // as we are deducting the min, scales are always positive float max_min = 0; for (int j = 0; j < QK_K/32; ++j) { @@ -2944,55 +2748,8 @@ void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict m1 <<= 2; m2 <<= 2; ql += 32; } -#else - float max_scale = 0, amax = 0; - for (int j = 0; j < QK_K/16; ++j) { - scales[j] = make_qx_quants(16, 16, x + 16*j, L + 16*j, 1, NULL); - float abs_scale = fabsf(scales[j]); - if (abs_scale > amax) { - amax = abs_scale; - max_scale = scales[j]; - } - } - - float iscale = -128.f/max_scale; - for (int j = 0; j < QK_K/16; ++j) { - int l = nearest_int(iscale*scales[j]); - y[i].scales[j] = MAX(-128, MIN(127, l)); - } - y[i].d = GGML_FP32_TO_FP16(1/iscale); - - for (int j = 0; j < QK_K/16; ++j) { - const float d = GGML_FP16_TO_FP32(y[i].d) * y[i].scales[j]; - if (!d) continue; - for (int ii = 0; ii < 16; ++ii) { - int l = nearest_int(x[16*j + ii]/d); - l = MAX(-16, MIN(15, l)); - L[16*j + ii] = l + 16; - } - } - - uint8_t * restrict qh = y[i].qh; - uint8_t * restrict ql = y[i].qs; - memset(qh, 0, QK_K/8); - - for (int j = 0; j < 32; ++j) { - int jm = j%8; - int is = j/8; - int l1 = L[j]; - if (l1 > 15) { - l1 -= 16; qh[jm] |= (1 << is); - } - int l2 = L[j + 32]; - if (l2 > 15) { - l2 -= 16; qh[jm] |= (1 << (4 + is)); - } - ql[j] = l1 | (l2 << 4); - } -#endif x += QK_K; - } } @@ -3001,12 +2758,9 @@ void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int6 const int64_t nb = k / QK_K; for (int i = 0; i < nb; i++) { - const uint8_t * ql = x[i].qs; const uint8_t * qh = x[i].qh; -#if QK_K == 256 - const float d = GGML_FP16_TO_FP32(x[i].d); const float min = GGML_FP16_TO_FP32(x[i].dmin); @@ -3023,21 +2777,6 @@ void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int6 ql += 32; is += 2; u1 <<= 2; u2 <<= 2; } -#else - float d = GGML_FP16_TO_FP32(x[i].d); - const int8_t * restrict s = x[i].scales; - for (int l = 0; l < 8; ++l) { - y[l+ 0] = d * s[0] * ((ql[l+ 0] & 0xF) - (qh[l] & 0x01 ? 0 : 16)); - y[l+ 8] = d * s[0] * ((ql[l+ 8] & 0xF) - (qh[l] & 0x02 ? 0 : 16)); - y[l+16] = d * s[1] * ((ql[l+16] & 0xF) - (qh[l] & 0x04 ? 0 : 16)); - y[l+24] = d * s[1] * ((ql[l+24] & 0xF) - (qh[l] & 0x08 ? 0 : 16)); - y[l+32] = d * s[2] * ((ql[l+ 0] >> 4) - (qh[l] & 0x10 ? 0 : 16)); - y[l+40] = d * s[2] * ((ql[l+ 8] >> 4) - (qh[l] & 0x20 ? 0 : 16)); - y[l+48] = d * s[3] * ((ql[l+16] >> 4) - (qh[l] & 0x40 ? 0 : 16)); - y[l+56] = d * s[3] * ((ql[l+24] >> 4) - (qh[l] & 0x80 ? 0 : 16)); - } - y += QK_K; -#endif } } @@ -3048,10 +2787,6 @@ void quantize_row_q5_K(const float * restrict x, void * restrict vy, int64_t k) } static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restrict y, int64_t n_per_row, const float * quant_weights) { -#if QK_K != 256 - (void)quant_weights; - quantize_row_q5_K_reference(x, y, n_per_row); -#else assert(n_per_row % QK_K == 0); const int64_t nb = n_per_row / QK_K; @@ -3142,7 +2877,6 @@ static void quantize_row_q5_K_impl(const float * restrict x, block_q5_K * restri x += QK_K; } -#endif } size_t quantize_q5_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { @@ -3215,7 +2949,6 @@ void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict uint8_t * restrict ql = y[i].ql; uint8_t * restrict qh = y[i].qh; -#if QK_K == 256 for (int j = 0; j < QK_K; j += 128) { for (int l = 0; l < 32; ++l) { const uint8_t q1 = L[j + l + 0] & 0xF; @@ -3229,19 +2962,8 @@ void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict ql += 64; qh += 32; } -#else - for (int l = 0; l < 32; ++l) { - const uint8_t q1 = L[l + 0] & 0xF; - const uint8_t q2 = L[l + 32] & 0xF; - ql[l] = q1 | (q2 << 4); - } - for (int l = 0; l < 16; ++l) { - qh[l] = (L[l] >> 4) | ((L[l + 16] >> 4) << 2) | ((L[l + 32] >> 4) << 4) | ((L[l + 48] >> 4) << 6); - } -#endif x += QK_K; - } } @@ -3250,14 +2972,12 @@ void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int6 const int64_t nb = k / QK_K; for (int i = 0; i < nb; i++) { - const float d = GGML_FP16_TO_FP32(x[i].d); const uint8_t * restrict ql = x[i].ql; const uint8_t * restrict qh = x[i].qh; const int8_t * restrict sc = x[i].scales; -#if QK_K == 256 for (int n = 0; n < QK_K; n += 128) { for (int l = 0; l < 32; ++l) { int is = l/16; @@ -3275,20 +2995,6 @@ void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int6 qh += 32; sc += 8; } -#else - for (int l = 0; l < 16; ++l) { - const int8_t q1 = (int8_t)((ql[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32; - const int8_t q2 = (int8_t)((ql[l+16] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32; - const int8_t q3 = (int8_t)((ql[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32; - const int8_t q4 = (int8_t)((ql[l+16] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32; - y[l+ 0] = d * sc[0] * q1; - y[l+16] = d * sc[1] * q2; - y[l+32] = d * sc[2] * q3; - y[l+48] = d * sc[3] * q4; - } - y += 64; -#endif - } } @@ -3299,10 +3005,6 @@ void quantize_row_q6_K(const float * restrict x, void * restrict vy, int64_t k) } static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restrict y, int64_t n_per_row, const float * quant_weights) { -#if QK_K != 256 - (void)quant_weights; - quantize_row_q6_K_reference(x, y, n_per_row); -#else assert(n_per_row % QK_K == 0); const int64_t nb = n_per_row / QK_K; @@ -3384,7 +3086,6 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri x += QK_K; } -#endif } size_t quantize_q6_K(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { @@ -3801,30 +3502,21 @@ void dequantize_row_iq1_m(const block_iq1_m * restrict x, float * restrict y, in float delta[4]; uint16_t idx[4]; -#if QK_K != 64 iq1m_scale_t scale; -#endif for (int i = 0; i < nb; i++) { const uint16_t * sc = (const uint16_t *)x[i].scales; -#if QK_K == 64 - const float d = GGML_FP16_TO_FP32(x[i].d); -#else scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); const float d = GGML_FP16_TO_FP32(scale.f16); -#endif + const uint8_t * qs = x[i].qs; const uint8_t * qh = x[i].qh; for (int ib = 0; ib < QK_K/32; ++ib) { -#if QK_K == 64 - const float dl1 = d * (2*((sc[ib/2] >> (8*(ib%2)+0)) & 0xf) + 1); - const float dl2 = d * (2*((sc[ib/2] >> (8*(ib%2)+4)) & 0xf) + 1); -#else const float dl1 = d * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1); const float dl2 = d * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1); -#endif + idx[0] = qs[0] | ((qh[0] << 8) & 0x700); idx[1] = qs[1] | ((qh[0] << 4) & 0x700); idx[2] = qs[2] | ((qh[1] << 8) & 0x700); @@ -3875,9 +3567,6 @@ void dequantize_row_iq4_nl(const block_iq4_nl * restrict x, float * restrict y, void dequantize_row_iq4_xs(const block_iq4_xs * restrict x, float * restrict y, int64_t k) { assert(k % QK_K == 0); -#if QK_K == 64 - dequantize_row_iq4_nl((const block_iq4_nl *)x, y, k); -#else const int64_t nb = k / QK_K; for (int i = 0; i < nb; i++) { @@ -3897,7 +3586,6 @@ void dequantize_row_iq4_xs(const block_iq4_xs * restrict x, float * restrict y, qs += 16; } } -#endif } //===================================== Q8_K ============================================== @@ -5849,7 +5537,6 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r #endif } -#if QK_K == 256 void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(nrc == 1); UNUSED(nrc); @@ -6433,410 +6120,6 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r #endif } -#else - -void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - - const block_q2_K * restrict x = vx; - const block_q8_K * restrict y = vy; - - const int nb = n / QK_K; - -#ifdef __ARM_NEON - const uint8x16_t m3 = vdupq_n_u8(0x3); - - const int32x4_t vzero = vdupq_n_s32(0); - - ggml_int8x16x4_t q2bytes; - - uint32_t aux32[2]; - const uint8_t * scales = (const uint8_t *)aux32; - - float sum = 0; - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); - - const uint8_t * restrict q2 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - const uint32_t * restrict sc = (const uint32_t *)x[i].scales; - - aux32[0] = sc[0] & 0x0f0f0f0f; - aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f; - - sum += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]); - - int isum1 = 0, isum2 = 0; - - const uint8x16_t q2bits = vld1q_u8(q2); - - const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); - - q2bytes.val[0] = vreinterpretq_s8_u8(vandq_u8(q2bits, m3)); - q2bytes.val[1] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 2), m3)); - q2bytes.val[2] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 4), m3)); - q2bytes.val[3] = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits, 6), m3)); - - isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[0], q8bytes.val[0])) * scales[0]; - isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[1], q8bytes.val[1])) * scales[1]; - isum1 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[2], q8bytes.val[2])) * scales[2]; - isum2 += vaddvq_s32(ggml_vdotq_s32(vzero, q2bytes.val[3], q8bytes.val[3])) * scales[3]; - - sum += d * (isum1 + isum2); - } - - *s = sum; - -#elif defined __AVX2__ - - const __m256i m3 = _mm256_set1_epi8(3); - - __m256 acc = _mm256_setzero_ps(); - - uint32_t ud, um; - const uint8_t * restrict db = (const uint8_t *)&ud; - const uint8_t * restrict mb = (const uint8_t *)&um; - - float summs = 0; - - // TODO: optimize this - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); - - const uint8_t * restrict q2 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const uint32_t * restrict sc = (const uint32_t *)x[i].scales; - ud = (sc[0] >> 0) & 0x0f0f0f0f; - um = (sc[0] >> 4) & 0x0f0f0f0f; - - int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3]; - summs += dmin * smin; - - const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); - const __m256i q2_0 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 2), q2bits), m3); - const __m256i q2_1 = _mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q2bits, 6), _mm_srli_epi16(q2bits, 4)), m3); - - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - const __m256i p0 = _mm256_maddubs_epi16(q2_0, q8_0); - const __m256i p1 = _mm256_maddubs_epi16(q2_1, q8_1); - - const __m256i p_0 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 0)); - const __m256i p_1 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p0, 1)); - const __m256i p_2 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 0)); - const __m256i p_3 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(p1, 1)); - - acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0), acc); - acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1), acc); - acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2), acc); - acc = _mm256_fmadd_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3), acc); - } - - *s = hsum_float_8(acc) + summs; - -#elif defined __AVX__ - - const __m128i m3 = _mm_set1_epi8(3); - - __m256 acc = _mm256_setzero_ps(); - - uint32_t ud, um; - const uint8_t * restrict db = (const uint8_t *)&ud; - const uint8_t * restrict mb = (const uint8_t *)&um; - - float summs = 0; - - // TODO: optimize this - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); - - const uint8_t * restrict q2 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const uint32_t * restrict sc = (const uint32_t *)x[i].scales; - ud = (sc[0] >> 0) & 0x0f0f0f0f; - um = (sc[0] >> 4) & 0x0f0f0f0f; - - int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3]; - summs += dmin * smin; - - const __m128i q2bits = _mm_loadu_si128((const __m128i*)q2); - const __m128i q2_0 = _mm_and_si128(q2bits, m3); - const __m128i q2_1 = _mm_and_si128(_mm_srli_epi16(q2bits, 2), m3); - const __m128i q2_2 = _mm_and_si128(_mm_srli_epi16(q2bits, 4), m3); - const __m128i q2_3 = _mm_and_si128(_mm_srli_epi16(q2bits, 6), m3); - - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - const __m128i p0 = _mm_maddubs_epi16(q2_0, _mm256_extractf128_si256(q8_0, 0)); - const __m128i p1 = _mm_maddubs_epi16(q2_1, _mm256_extractf128_si256(q8_0, 1)); - const __m128i p2 = _mm_maddubs_epi16(q2_2, _mm256_extractf128_si256(q8_1, 0)); - const __m128i p3 = _mm_maddubs_epi16(q2_3, _mm256_extractf128_si256(q8_1, 1)); - - const __m256i p_0 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p0, p0)), _mm_cvtepi16_epi32(p0)); - const __m256i p_1 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p1, p1)), _mm_cvtepi16_epi32(p1)); - const __m256i p_2 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p2, p2)), _mm_cvtepi16_epi32(p2)); - const __m256i p_3 = MM256_SET_M128I(_mm_cvtepi16_epi32(_mm_unpackhi_epi64(p3, p3)), _mm_cvtepi16_epi32(p3)); - - acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[0]), _mm256_cvtepi32_ps(p_0)), acc); - acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[1]), _mm256_cvtepi32_ps(p_1)), acc); - acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[2]), _mm256_cvtepi32_ps(p_2)), acc); - acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d * db[3]), _mm256_cvtepi32_ps(p_3)), acc); - } - - *s = hsum_float_8(acc) + summs; - -#elif defined __riscv_v_intrinsic - - uint32_t aux32[2]; - const uint8_t * scales = (const uint8_t *)aux32; - - float sumf = 0; - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); - - const uint8_t * restrict q2 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - const uint32_t * restrict sc = (const uint32_t *)x[i].scales; - - aux32[0] = sc[0] & 0x0f0f0f0f; - aux32[1] = (sc[0] >> 4) & 0x0f0f0f0f; - - sumf += dmin * (scales[4] * y[i].bsums[0] + scales[5] * y[i].bsums[1] + scales[6] * y[i].bsums[2] + scales[7] * y[i].bsums[3]); - - int isum1 = 0; - int isum2 = 0; - - size_t vl = 16; - - vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1); - - // load Q2 - vuint8mf2_t q2_x = __riscv_vle8_v_u8mf2(q2, vl); - - vint8mf2_t q2_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q2_x, 0x03, vl)); - vint8mf2_t q2_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x2, vl), 0x03 , vl)); - vint8mf2_t q2_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x4, vl), 0x03 , vl)); - vint8mf2_t q2_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q2_x, 0x6, vl), 0x03 , vl)); - - // load Q8, and take product with Q2 - vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q2_0, __riscv_vle8_v_i8mf2(q8, vl), vl); - vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q2_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl); - vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q2_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl); - vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q2_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl); - - vint16m1_t vs_0 = __riscv_vredsum_vs_i16m1_i16m1(p0, vzero, vl); - vint16m1_t vs_1 = __riscv_vredsum_vs_i16m1_i16m1(p1, vzero, vl); - vint16m1_t vs_2 = __riscv_vredsum_vs_i16m1_i16m1(p2, vzero, vl); - vint16m1_t vs_3 = __riscv_vredsum_vs_i16m1_i16m1(p3, vzero, vl); - - isum1 += __riscv_vmv_x_s_i16m1_i16(vs_0) * scales[0]; - isum2 += __riscv_vmv_x_s_i16m1_i16(vs_1) * scales[1]; - isum1 += __riscv_vmv_x_s_i16m1_i16(vs_2) * scales[2]; - isum2 += __riscv_vmv_x_s_i16m1_i16(vs_3) * scales[3]; - - sumf += d * (isum1 + isum2); - - } - - *s = sumf; - - -#elif defined(__POWER9_VECTOR__) - const vector signed char lowMask = vec_splats((signed char)0x3); - const vector signed char lowScaleMask = vec_splats((signed char)0xF); - const vector unsigned char v2 = vec_splats((unsigned char)0x2); - const vector unsigned char v4 = vec_splats((unsigned char)0x4); - const vector unsigned char v6 = vec_splats((unsigned char)0x6); - - vector float vsumf0 = vec_splats(0.0f); - vector float vsumf1 = vec_splats(0.0f); - vector float vsumf2 = vec_splats(0.0f); - vector float vsumf3 = vec_splats(0.0f); - -#pragma GCC unroll 2 - for (int i = 0; i < nb; ++i) { - __builtin_prefetch(x[i].qs, 0, 1); - __builtin_prefetch(y[i].qs, 0, 1); - - vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); - vector float vyd = vec_splats(y[i].d); - vector float vd = vec_mul(vxd, vyd); - - vector float vxmin = vec_splats(GGML_FP16_TO_FP32(x[i].dmin)); - vector float vdmin = vec_mul(vxmin, vyd); - - vector signed short q8ysums0 = vec_xl_len(y[i].bsums, 8); - - vector signed char q2xmins = (vector signed char)vec_xl_len(x[i].scales, 4); - vector signed char vscales = vec_and(q2xmins, lowScaleMask); - - q2xmins = vec_sr(q2xmins, v4); - vector signed short q2xmins0 = vec_unpackh((vector signed char)q2xmins); - - vector signed int prod0 = vec_mule(q2xmins0, q8ysums0); - vector signed int prod1 = vec_mulo(q2xmins0, q8ysums0); - - vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vdmin, vsumf0); - vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vdmin, vsumf1); - - vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs); - vector signed char q2x00 = vec_and(qxs0, lowMask); - vector signed char q2x01 = vec_and(vec_sr(qxs0, v2), lowMask); - vector signed char q2x02 = vec_and(vec_sr(qxs0, v4), lowMask); - vector signed char q2x03 = vec_and(vec_sr(qxs0, v6), lowMask); - - vector signed char q8y00 = vec_xl( 0, y[i].qs); - vector signed char q8y01 = vec_xl( 16, y[i].qs); - vector signed char q8y02 = vec_xl( 32, y[i].qs); - vector signed char q8y03 = vec_xl( 48, y[i].qs); - - vector signed short qv0 = vec_add(vec_mule(q2x00, q8y00), vec_mulo(q2x00, q8y00)); - vector signed short qv1 = vec_add(vec_mule(q2x01, q8y01), vec_mulo(q2x01, q8y01)); - vector signed short qv2 = vec_add(vec_mule(q2x02, q8y02), vec_mulo(q2x02, q8y02)); - vector signed short qv3 = vec_add(vec_mule(q2x03, q8y03), vec_mulo(q2x03, q8y03)); - - vector signed short vscales_h = vec_unpackh(vscales); - vector signed short vs0 = vec_splat(vscales_h, 0); - vector signed short vs1 = vec_splat(vscales_h, 1); - vector signed short vs2 = vec_splat(vscales_h, 2); - vector signed short vs3 = vec_splat(vscales_h, 3); - - vector signed int vsumi0 = vec_add(vec_mule(qv0, vs0), vec_mulo(qv0, vs0)); - vector signed int vsumi1 = vec_add(vec_mule(qv1, vs1), vec_mulo(qv1, vs1)); - vector signed int vsumi2 = vec_add(vec_mule(qv2, vs2), vec_mulo(qv2, vs2)); - vector signed int vsumi3 = vec_add(vec_mule(qv3, vs3), vec_mulo(qv3, vs3)); - - vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); - vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); - vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); - vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); - } - - vsumf0 = vec_add(vsumf0, vsumf2); - vsumf1 = vec_add(vsumf1, vsumf3); - - vsumf0 = vec_add(vsumf0, vsumf1); - - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); - - *s = vec_extract(vsumf0, 0); - -#elif defined __loongarch_asx - - const __m256i m3 = __lasx_xvreplgr2vr_b(3); - - __m256 acc = (__m256)__lasx_xvldi(0); - - uint32_t ud, um; - const uint8_t * restrict db = (const uint8_t *)&ud; - const uint8_t * restrict mb = (const uint8_t *)&um; - - float summs = 0; - - // TODO: optimize this - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); - - const uint8_t * restrict q2 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const uint32_t * restrict sc = (const uint32_t *)x[i].scales; - ud = (sc[0] >> 0) & 0x0f0f0f0f; - um = (sc[0] >> 4) & 0x0f0f0f0f; - - int32_t smin = mb[0] * y[i].bsums[0] + mb[1] * y[i].bsums[1] + mb[2] * y[i].bsums[2] + mb[3] * y[i].bsums[3]; - summs += dmin * smin; - - const __m128i q2bits = __lsx_vld((const __m128i*)q2, 0); - const __m256i q2_0 = __lasx_xvand_v(lasx_insertf128(__lsx_vsrli_h(q2bits, 2), q2bits), m3); - const __m256i q2_1 = __lasx_xvand_v(lasx_insertf128(__lsx_vsrli_h(q2bits, 6), __lsx_vsrli_h(q2bits, 4)), m3); - - const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0); - const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0); - - const __m256i p0 = lasx_maddubs_h(q2_0, q8_0); - const __m256i p1 = lasx_maddubs_h(q2_1, q8_1); - - const __m256i p_0 = lasx_ext16_32(lasx_extracti128(p0, 0)); - const __m256i p_1 = lasx_ext16_32(lasx_extracti128(p0, 1)); - const __m256i p_2 = lasx_ext16_32(lasx_extracti128(p1, 0)); - const __m256i p_3 = lasx_ext16_32(lasx_extracti128(p1, 1)); - - ft_union t0, t1, t2, t3; - t0.f = d * db[0]; - t1.f = d * db[1]; - t2.f = d * db[2]; - t3.f = d * db[3]; - acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t0.i), __lasx_xvffint_s_w(p_0), acc); - acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t1.i), __lasx_xvffint_s_w(p_1), acc); - acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t2.i), __lasx_xvffint_s_w(p_2), acc); - acc = __lasx_xvfmadd_s(__lasx_xvreplgr2vr_w(t3.i), __lasx_xvffint_s_w(p_3), acc); - } - - *s = hsum_float_8(acc) + summs; - -#else - - float sumf = 0; - - int isum[QK_K/16]; - - for (int i = 0; i < nb; ++i) { - - const uint8_t * q2 = x[i].qs; - const int8_t * q8 = y[i].qs; - const uint8_t * sc = x[i].scales; - - int summs = 0; - for (int j = 0; j < QK_K/16; ++j) { - summs += y[i].bsums[j] * (sc[j] >> 4); - } - - const float dall = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin); - - memset(isum, 0, (QK_K/16)*sizeof(int)); - for (int l = 0; l < 16; ++l) { - isum[0] += q8[l+ 0] * ((q2[l] >> 0) & 3); - isum[1] += q8[l+16] * ((q2[l] >> 2) & 3); - isum[2] += q8[l+32] * ((q2[l] >> 4) & 3); - isum[3] += q8[l+48] * ((q2[l] >> 6) & 3); - } - for (int l = 0; l < QK_K/16; ++l) { - isum[l] *= (sc[l] & 0xF); - } - sumf += dall * (isum[0] + isum[1] + isum[2] + isum[3]) - dmin * summs; - } - *s = sumf; -#endif -} -#endif - -#if QK_K == 256 void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -7616,512 +6899,6 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r } -#else - -void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { - assert(n % QK_K == 0); - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - - const block_q3_K * restrict x = vx; - const block_q8_K * restrict y = vy; - - const int nb = n / QK_K; - -#ifdef __ARM_NEON - const int32x4_t vzero = vdupq_n_s32(0); - - const uint8x16_t m3b = vdupq_n_u8(0x3); - const uint8x16_t mh = vdupq_n_u8(4); - - ggml_int8x16x4_t q3bytes; - - uint16_t aux16[2]; - int8_t * scales = (int8_t *)aux16; - - float sum = 0; - - for (int i = 0; i < nb; ++i) { - - ggml_uint8x16x4_t q3h; - - const uint8x8_t hbits = vld1_u8(x[i].hmask); - const uint8x16_t q3bits = vld1q_u8(x[i].qs); - const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(y[i].qs); - - const uint16_t a = *(const uint16_t *)x[i].scales; - aux16[0] = a & 0x0f0f; - aux16[1] = (a >> 4) & 0x0f0f; - - for (int j = 0; j < 4; ++j) scales[j] -= 8; - - int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]); - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const uint8x16_t htmp = vcombine_u8(hbits, vshr_n_u8(hbits, 1)); - q3h.val[0] = vandq_u8(mh, vshlq_n_u8(htmp, 2)); - q3h.val[1] = vandq_u8(mh, htmp); - q3h.val[2] = vandq_u8(mh, vshrq_n_u8(htmp, 2)); - q3h.val[3] = vandq_u8(mh, vshrq_n_u8(htmp, 4)); - - q3bytes.val[0] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q3bits, m3b), q3h.val[0])); - q3bytes.val[1] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 2), m3b), q3h.val[1])); - q3bytes.val[2] = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(vshrq_n_u8(q3bits, 4), m3b), q3h.val[2])); - q3bytes.val[3] = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q3bits, 6), q3h.val[3])); - - isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[0], q8bytes.val[0])) * scales[0]; - isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[1], q8bytes.val[1])) * scales[2]; - isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[2], q8bytes.val[2])) * scales[1]; - isum += vaddvq_s32(ggml_vdotq_s32(vzero, q3bytes.val[3], q8bytes.val[3])) * scales[3]; - - sum += d * isum; - - } - - *s = sum; - -#elif defined __AVX2__ - - const __m256i m3 = _mm256_set1_epi8(3); - const __m256i m1 = _mm256_set1_epi8(1); - - __m256 acc = _mm256_setzero_ps(); - - uint64_t aux64; - - uint16_t aux16[2]; - const int8_t * aux8 = (const int8_t *)aux16; - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q3 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const uint16_t a = *(const uint16_t *)x[i].scales; - aux16[0] = a & 0x0f0f; - aux16[1] = (a >> 4) & 0x0f0f; - - const __m256i scale_0 = MM256_SET_M128I(_mm_set1_epi16(aux8[2] - 8), _mm_set1_epi16(aux8[0] - 8)); - const __m256i scale_1 = MM256_SET_M128I(_mm_set1_epi16(aux8[3] - 8), _mm_set1_epi16(aux8[1] - 8)); - - memcpy(&aux64, x[i].hmask, 8); - - const __m128i haux = _mm_set_epi64x(aux64 >> 1, aux64 >> 0); - __m256i q3h_0 = MM256_SET_M128I(_mm_srli_epi16(haux, 2), haux); - __m256i q3h_1 = _mm256_srli_epi16(q3h_0, 4); - q3h_0 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_0, m1), 2); - q3h_1 = _mm256_slli_epi16(_mm256_andnot_si256(q3h_1, m1), 2); - - // load low 2 bits - const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3); - - // prepare low and high bits - const __m256i q3aux = MM256_SET_M128I(_mm_srli_epi16(q3bits, 2), q3bits); - const __m256i q3l_0 = _mm256_and_si256(q3aux, m3); - const __m256i q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3aux, 4), m3); - - // load Q8 quants - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm256_maddubs_epi16, - // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set, - // and 2 if the high bit was set) - const __m256i q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0); - const __m256i q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1); - - __m256i p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0); - __m256i p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1); - - p16_0 = _mm256_sub_epi16(p16_0, q8s_0); - p16_1 = _mm256_sub_epi16(p16_1, q8s_1); - - // multiply with scales - p16_0 = _mm256_madd_epi16(scale_0, p16_0); - p16_1 = _mm256_madd_epi16(scale_1, p16_1); - - p16_0 = _mm256_add_epi32(p16_0, p16_1); - - // multiply with block scale and accumulate - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16_0), acc); - - } - - *s = hsum_float_8(acc); - -#elif defined __AVX__ - - const __m128i m3 = _mm_set1_epi8(3); - const __m128i m1 = _mm_set1_epi8(1); - - __m256 acc = _mm256_setzero_ps(); - - uint64_t aux64; - - uint16_t aux16[2]; - const int8_t * aux8 = (const int8_t *)aux16; - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q3 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const uint16_t a = *(const uint16_t *)x[i].scales; - aux16[0] = a & 0x0f0f; - aux16[1] = (a >> 4) & 0x0f0f; - - const __m128i scale_0 = _mm_set1_epi16(aux8[0] - 8); - const __m128i scale_1 = _mm_set1_epi16(aux8[2] - 8); - const __m128i scale_2 = _mm_set1_epi16(aux8[1] - 8); - const __m128i scale_3 = _mm_set1_epi16(aux8[3] - 8); - - memcpy(&aux64, x[i].hmask, 8); - - __m128i q3h_0 = _mm_set_epi64x(aux64 >> 1, aux64 >> 0); - __m128i q3h_1 = _mm_srli_epi16(q3h_0, 2); - __m128i q3h_2 = _mm_srli_epi16(q3h_0, 4); - __m128i q3h_3 = _mm_srli_epi16(q3h_0, 6); - q3h_0 = _mm_slli_epi16(_mm_andnot_si128(q3h_0, m1), 2); - q3h_1 = _mm_slli_epi16(_mm_andnot_si128(q3h_1, m1), 2); - q3h_2 = _mm_slli_epi16(_mm_andnot_si128(q3h_2, m1), 2); - q3h_3 = _mm_slli_epi16(_mm_andnot_si128(q3h_3, m1), 2); - - // load low 2 bits - const __m128i q3bits = _mm_loadu_si128((const __m128i*)q3); - - // prepare low and high bits - const __m128i q3l_0 = _mm_and_si128(q3bits, m3); - const __m128i q3l_1 = _mm_and_si128(_mm_srli_epi16(q3bits, 2), m3); - const __m128i q3l_2 = _mm_and_si128(_mm_srli_epi16(q3bits, 4), m3); - const __m128i q3l_3 = _mm_and_si128(_mm_srli_epi16(q3bits, 6), m3); - - // load Q8 quants - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use _mm_maddubs_epi16, - // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set, - // and 2 if the high bit was set) - const __m128i q8s_0 = _mm_maddubs_epi16(q3h_0, _mm256_extractf128_si256(q8_0, 0)); - const __m128i q8s_1 = _mm_maddubs_epi16(q3h_1, _mm256_extractf128_si256(q8_0, 1)); - const __m128i q8s_2 = _mm_maddubs_epi16(q3h_2, _mm256_extractf128_si256(q8_1, 0)); - const __m128i q8s_3 = _mm_maddubs_epi16(q3h_3, _mm256_extractf128_si256(q8_1, 1)); - - __m128i p16_0 = _mm_maddubs_epi16(q3l_0, _mm256_extractf128_si256(q8_0, 0)); - __m128i p16_1 = _mm_maddubs_epi16(q3l_1, _mm256_extractf128_si256(q8_0, 1)); - __m128i p16_2 = _mm_maddubs_epi16(q3l_2, _mm256_extractf128_si256(q8_1, 0)); - __m128i p16_3 = _mm_maddubs_epi16(q3l_3, _mm256_extractf128_si256(q8_1, 1)); - - p16_0 = _mm_sub_epi16(p16_0, q8s_0); - p16_1 = _mm_sub_epi16(p16_1, q8s_1); - p16_2 = _mm_sub_epi16(p16_2, q8s_2); - p16_3 = _mm_sub_epi16(p16_3, q8s_3); - - // multiply with scales - p16_0 = _mm_madd_epi16(scale_0, p16_0); - p16_1 = _mm_madd_epi16(scale_1, p16_1); - p16_2 = _mm_madd_epi16(scale_2, p16_2); - p16_3 = _mm_madd_epi16(scale_3, p16_3); - - p16_0 = _mm_add_epi32(p16_0, p16_2); - p16_1 = _mm_add_epi32(p16_1, p16_3); - __m256i p16 = MM256_SET_M128I(p16_1, p16_0); - - // multiply with block scale and accumulate - acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(p16)), acc); - - } - - *s = hsum_float_8(acc); - -#elif defined __riscv_v_intrinsic - - uint16_t aux16[2]; - int8_t * scales = (int8_t *)aux16; - - float sumf = 0; - - for (int i = 0; i < nb; ++i) { - - const uint8_t * restrict q3 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const uint16_t a = *(const uint16_t *)x[i].scales; - aux16[0] = a & 0x0f0f; - aux16[1] = (a >> 4) & 0x0f0f; - - for (int j = 0; j < 4; ++j) scales[j] -= 8; - - int32_t isum = -4*(scales[0] * y[i].bsums[0] + scales[2] * y[i].bsums[1] + scales[1] * y[i].bsums[2] + scales[3] * y[i].bsums[3]); - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); - - // load qh - vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(x[i].hmask, 8); - vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8)); - - size_t vl = 16; - - // extend and combine both qh_x1 and qh_x2 - vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl); - - vuint8mf2_t qh_0 = __riscv_vand_vx_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl); - vuint8mf2_t qh_1 = __riscv_vand_vx_u8mf2(qh_x, 0x4, vl); - vuint8mf2_t qh_2 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl), 0x4, vl); - vuint8mf2_t qh_3 = __riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), 0x4, vl); - - // load Q3 - vuint8mf2_t q3_x = __riscv_vle8_v_u8mf2(q3, vl); - - vuint8mf2_t q3h_0 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(q3_x, 0x3, vl), qh_0, vl); - vuint8mf2_t q3h_1 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 2, vl), 0x3, vl), qh_1, vl); - vuint8mf2_t q3h_2 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 4, vl), 0x3, vl), qh_2, vl); - vuint8mf2_t q3h_3 = __riscv_vor_vv_u8mf2(__riscv_vsrl_vx_u8mf2(q3_x, 0x6, vl), qh_3, vl); - - vint8mf2_t q3_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_0); - vint8mf2_t q3_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_1); - vint8mf2_t q3_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_2); - vint8mf2_t q3_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(q3h_3); - - // load Q8 and take product with Q3 - vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q3_0, __riscv_vle8_v_i8mf2(q8, vl), vl); - vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q3_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl); - vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q3_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl); - vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q3_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl); - - vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl); - vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl); - vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl); - vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl); - - isum += __riscv_vmv_x_s_i32m1_i32(vs_0) * scales[0]; - isum += __riscv_vmv_x_s_i32m1_i32(vs_1) * scales[2]; - isum += __riscv_vmv_x_s_i32m1_i32(vs_2) * scales[1]; - isum += __riscv_vmv_x_s_i32m1_i32(vs_3) * scales[3]; - - sumf += d * isum; - - } - - *s = sumf; - -#elif defined(__POWER9_VECTOR__) - const vector signed char lowMask = vec_splats((signed char)0x3); - const vector signed char v1 = vec_splats((signed char)0x1); - const vector unsigned char v2 = vec_splats((unsigned char)0x2); - const vector unsigned char v4 = vec_splats((unsigned char)0x4); - const vector unsigned char v6 = vec_splats((unsigned char)0x6); - const vector signed char off = vec_splats((signed char)0x8); - - vector float vsumf0 = vec_splats(0.0f); - vector float vsumf1 = vec_splats(0.0f); - vector float vsumf2 = vec_splats(0.0f); - vector float vsumf3 = vec_splats(0.0f); - -#pragma GCC unroll 2 - for (int i = 0; i < nb; ++i) { - __builtin_prefetch(x[i].qs, 0, 1); - __builtin_prefetch(y[i].qs, 0, 1); - - vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); - vector float vyd = vec_splats(y[i].d); - vector float vd = vec_mul(vxd, vyd); - - uint16_t aux16[2]; - int8_t * scales = (int8_t *)aux16; - - const uint16_t a = *(const uint16_t *)x[i].scales; - aux16[0] = a & 0x0f0f; - aux16[1] = (a >> 4) & 0x0f0f; - - vector signed char vscales = (vector signed char)vec_xl_len(scales, 8); - vector signed char qxhs0 = (vector signed char)vec_xl_len(x[i].hmask, 8); - qxhs0 = vec_or(qxhs0, vec_sr(vec_sld(qxhs0, qxhs0, 8), (vector unsigned char)v1)); - - vscales = vec_sub(vscales, off); - - vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs); - vector signed char qxs00 = vec_and(qxs0, lowMask); - vector signed char qxs01 = vec_and(vec_sr(qxs0, v2), lowMask); - vector signed char qxs10 = vec_and(vec_sr(qxs0, v4), lowMask); - vector signed char qxs11 = vec_and(vec_sr(qxs0, v6), lowMask); - - //the 3rd bit - vector signed char qxh00 = vec_sl(vec_andc(v1, qxhs0), v2); - vector signed char qxh01 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v2)), v2); - vector signed char qxh02 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v4)), v2); - vector signed char qxh03 = vec_sl(vec_andc(v1, vec_sr(qxhs0, v6)), v2); - qxhs0 = vec_sr(qxhs0, v4); - - vector signed char q3x00 = vec_sub(qxs00, qxh00); - vector signed char q3x01 = vec_sub(qxs01, qxh01); - vector signed char q3x10 = vec_sub(qxs10, qxh02); - vector signed char q3x11 = vec_sub(qxs11, qxh03); - - vector signed char q8y00 = vec_xl( 0, y[i].qs); - vector signed char q8y01 = vec_xl( 16, y[i].qs); - vector signed char q8y10 = vec_xl( 32, y[i].qs); - vector signed char q8y11 = vec_xl( 48, y[i].qs); - - vector signed short vscales_h = vec_unpackh(vscales); - vector signed short vs0 = vec_splat(vscales_h, 0); - vector signed short vs1 = vec_splat(vscales_h, 1); - vector signed short vs2 = vec_splat(vscales_h, 2); - vector signed short vs3 = vec_splat(vscales_h, 3); - - vector signed short qv00 = vec_add(vec_mule(q3x00, q8y00), vec_mulo(q3x00, q8y00)); - vector signed short qv10 = vec_add(vec_mule(q3x10, q8y10), vec_mulo(q3x10, q8y10)); - vector signed short qv01 = vec_add(vec_mule(q3x01, q8y01), vec_mulo(q3x01, q8y01)); - vector signed short qv11 = vec_add(vec_mule(q3x11, q8y11), vec_mulo(q3x11, q8y11)); - - vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); - vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1)); - vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2)); - vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3)); - - vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); - vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); - vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); - vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); - } - - vsumf0 = vec_add(vsumf0, vsumf2); - vsumf1 = vec_add(vsumf1, vsumf3); - - vsumf0 = vec_add(vsumf0, vsumf1); - - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); - - *s = vec_extract(vsumf0, 0); - -#elif defined __loongarch_asx - - const __m256i m3 = __lasx_xvreplgr2vr_b(3); - const __m256i m1 = __lasx_xvreplgr2vr_b(1); - - __m256 acc = (__m256)__lasx_xvldi(0); - - uint64_t aux64; - - uint16_t aux16[2]; - const int8_t * aux8 = (const int8_t *)aux16; - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q3 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - const __m256i scale_0 = lasx_insertf128(__lasx_xvreplgr2vr_h(aux8[2] - 8), __lasx_xvreplgr2vr_h(aux8[0] - 8)); - const __m256i scale_1 = lasx_insertf128(__lasx_xvreplgr2vr_h(aux8[3] - 8), __lasx_xvreplgr2vr_h(aux8[1] - 8)); - - memcpy(&aux64, x[i].hmask, 8); - - __m128i haux = __lsx_vinsgr2vr_d(haux, aux64, 0); - haux = __lsx_vinsgr2vr_d(haux, aux64 >> 1, 1); - __m256i q3h_0 = lasx_insertf128(__lsx_vsrli_h(haux, 2), haux); - __m256i q3h_1 = __lasx_xvsrli_h(q3h_0, 4); - q3h_0 = __lasx_xvslli_h(__lasx_xvandn_v(q3h_0, m1), 2); - q3h_1 = __lasx_xvslli_h(__lasx_xvandn_v(q3h_1, m1), 2); - - // load low 2 bits - const __m128i q3bits = __lsx_vld((const __m128i*)q3, 0); - - // prepare low and high bits - const __m256i q3aux = lasx_insertf128(__lsx_vsrli_h(q3bits, 2), q3bits); - const __m256i q3l_0 = __lasx_xvand_v(q3aux, m3); - const __m256i q3l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q3aux, 4), m3); - - // load Q8 quants - const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0); - const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0); - - // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use lasx_maddubs_h, - // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set, - // and 2 if the high bit was set) - const __m256i q8s_0 = lasx_maddubs_h(q3h_0, q8_0); - const __m256i q8s_1 = lasx_maddubs_h(q3h_1, q8_1); - - __m256i p16_0 = lasx_maddubs_h(q3l_0, q8_0); - __m256i p16_1 = lasx_maddubs_h(q3l_1, q8_1); - - p16_0 = __lasx_xvsub_h(p16_0, q8s_0); - p16_1 = __lasx_xvsub_h(p16_1, q8s_1); - - // multiply with scales - p16_0 = lasx_madd_h(scale_0, p16_0); - p16_1 = lasx_madd_h(scale_1, p16_1); - - p16_0 = __lasx_xvadd_w(p16_0, p16_1); - - // multiply with block scale and accumulate - acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(p16_0), acc); - } - - *s = hsum_float_8(acc); - -#else - - int8_t aux8[QK_K]; - int16_t aux16[8]; - float sums [8]; - int32_t aux32[8]; - int32_t scales[4]; - memset(sums, 0, 8*sizeof(float)); - - float sumf = 0; - for (int i = 0; i < nb; ++i) { - const uint8_t * restrict q3 = x[i].qs; - const uint8_t * restrict hm = x[i].hmask; - const int8_t * restrict q8 = y[i].qs; - int8_t * restrict a = aux8; - for (int l = 0; l < 8; ++l) { - a[l+ 0] = (int8_t)((q3[l+0] >> 0) & 3) - (hm[l] & 0x01 ? 0 : 4); - a[l+ 8] = (int8_t)((q3[l+8] >> 0) & 3) - (hm[l] & 0x02 ? 0 : 4); - a[l+16] = (int8_t)((q3[l+0] >> 2) & 3) - (hm[l] & 0x04 ? 0 : 4); - a[l+24] = (int8_t)((q3[l+8] >> 2) & 3) - (hm[l] & 0x08 ? 0 : 4); - a[l+32] = (int8_t)((q3[l+0] >> 4) & 3) - (hm[l] & 0x10 ? 0 : 4); - a[l+40] = (int8_t)((q3[l+8] >> 4) & 3) - (hm[l] & 0x20 ? 0 : 4); - a[l+48] = (int8_t)((q3[l+0] >> 6) & 3) - (hm[l] & 0x40 ? 0 : 4); - a[l+56] = (int8_t)((q3[l+8] >> 6) & 3) - (hm[l] & 0x80 ? 0 : 4); - } - - scales[0] = (x[i].scales[0] & 0xF) - 8; - scales[1] = (x[i].scales[0] >> 4) - 8; - scales[2] = (x[i].scales[1] & 0xF) - 8; - scales[3] = (x[i].scales[1] >> 4) - 8; - - memset(aux32, 0, 8*sizeof(int32_t)); - for (int j = 0; j < QK_K/16; ++j) { - for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l]; - q8 += 8; a += 8; - for (int l = 0; l < 8; ++l) aux16[l] += q8[l] * a[l]; - q8 += 8; a += 8; - for (int l = 0; l < 8; ++l) aux32[l] += scales[j] * aux16[l]; - } - const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; - for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l]; - } - for (int l = 0; l < 8; ++l) sumf += sums[l]; - *s = sumf; - -#endif - -} -#endif - -#if QK_K == 256 void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -8679,381 +7456,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = sumf; #endif } -#else -void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { - assert(n % QK_K == 0); - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - const block_q4_K * restrict x = vx; - const block_q8_K * restrict y = vy; - - const int nb = n / QK_K; - -#ifdef __ARM_NEON - const uint8x16_t m4b = vdupq_n_u8(0xf); - - const int32x4_t mzero = vdupq_n_s32(0); - - float sumf = 0; - - ggml_int8x16x2_t q4bytes; - ggml_int8x16x4_t q8bytes; - - float sum_mins = 0.f; - - uint16_t aux16[2]; - const uint8_t * restrict scales = (const uint8_t *)aux16; - - for (int i = 0; i < nb; ++i) { - - const uint8_t * restrict q4 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const uint16_t * restrict a = (const uint16_t *)x[i].scales; - aux16[0] = a[0] & 0x0f0f; - aux16[1] = (a[0] >> 4) & 0x0f0f; - - const int32_t summi = scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3]); - sum_mins += y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * summi; - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]); - - const ggml_uint8x16x2_t q4bits = ggml_vld1q_u8_x2(q4); - - q8bytes = ggml_vld1q_s8_x4(q8); - q4bytes.val[0] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[0], m4b)); - q4bytes.val[1] = vreinterpretq_s8_u8(vandq_u8 (q4bits.val[1], m4b)); - - const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[0]), q4bytes.val[1], q8bytes.val[1]); - const int32_t sumi1 = vaddvq_s32(p1) * scales[0]; - - q4bytes.val[0] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[0], 4)); - q4bytes.val[1] = vreinterpretq_s8_u8(vshrq_n_u8(q4bits.val[1], 4)); - - const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(mzero, q4bytes.val[0], q8bytes.val[2]), q4bytes.val[1], q8bytes.val[3]); - const int32_t sumi2 = vaddvq_s32(p2) * scales[1]; - - sumf += d * (sumi1 + sumi2); - } - - *s = sumf - sum_mins; - -#elif defined __AVX2__ - - const __m256i m4 = _mm256_set1_epi8(0xF); - - __m256 acc = _mm256_setzero_ps(); - - float summs = 0; - - uint16_t aux16[2]; - const uint8_t * scales = (const uint8_t *)aux16; - - for (int i = 0; i < nb; ++i) { - - const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d; - const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d; - const __m256 vd = _mm256_set1_ps(d); - - const uint16_t * a = (const uint16_t *)x[i].scales; - aux16[0] = a[0] & 0x0f0f; - aux16[1] = (a[0] >> 4) & 0x0f0f; - - summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3])); - - const uint8_t * restrict q4 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); - const __m256i q4l = _mm256_and_si256(q4bits, m4); - const __m256i q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4); - - const __m256i q8l = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8h = _mm256_loadu_si256((const __m256i*)(q8+32)); - - const __m256i p16l = _mm256_maddubs_epi16(q4l, q8l); - const __m256i p16h = _mm256_maddubs_epi16(q4h, q8h); - - const __m256i p32l = _mm256_madd_epi16(_mm256_set1_epi16(scales[0]), p16l); - acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32l), acc); - - const __m256i p32h = _mm256_madd_epi16(_mm256_set1_epi16(scales[1]), p16h); - acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(p32h), acc); - - } - - *s = hsum_float_8(acc) - summs; - -#elif defined __AVX__ - - const __m128i m4 = _mm_set1_epi8(0xF); - - __m256 acc = _mm256_setzero_ps(); - - float summs = 0; - - uint16_t aux16[2]; - const uint8_t * scales = (const uint8_t *)aux16; - - for (int i = 0; i < nb; ++i) { - - const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d; - const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d; - const __m256 vd = _mm256_set1_ps(d); - - const uint16_t * a = (const uint16_t *)x[i].scales; - aux16[0] = a[0] & 0x0f0f; - aux16[1] = (a[0] >> 4) & 0x0f0f; - - summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3])); - - const uint8_t * restrict q4 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const __m256i q4bits = _mm256_loadu_si256((const __m256i*)q4); - const __m128i q4bits_0 = _mm256_extractf128_si256(q4bits, 0); - const __m128i q4bits_1 = _mm256_extractf128_si256(q4bits, 1); - const __m128i q4_0 = _mm_and_si128(q4bits_0, m4); - const __m128i q4_1 = _mm_and_si128(q4bits_1, m4); - const __m128i q4_2 = _mm_and_si128(_mm_srli_epi16(q4bits_0, 4), m4); - const __m128i q4_3 = _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4); - - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - const __m128i p16_0 = _mm_maddubs_epi16(q4_0, _mm256_extractf128_si256(q8_0, 0)); - const __m128i p16_1 = _mm_maddubs_epi16(q4_1, _mm256_extractf128_si256(q8_0, 1)); - const __m128i p16_2 = _mm_maddubs_epi16(q4_2, _mm256_extractf128_si256(q8_1, 0)); - const __m128i p16_3 = _mm_maddubs_epi16(q4_3, _mm256_extractf128_si256(q8_1, 1)); - - const __m128i p32_0 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_0); - const __m128i p32_1 = _mm_madd_epi16(_mm_set1_epi16(scales[0]), p16_1); - acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_1, p32_0))), acc); - - const __m128i p32_2 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_2); - const __m128i p32_3 = _mm_madd_epi16(_mm_set1_epi16(scales[1]), p16_3); - acc = _mm256_add_ps(_mm256_mul_ps(vd, _mm256_cvtepi32_ps(MM256_SET_M128I(p32_3, p32_2))), acc); - - } - - *s = hsum_float_8(acc) - summs; - -#elif defined __riscv_v_intrinsic - - uint16_t s16[2]; - const uint8_t * restrict scales = (const uint8_t *)s16; - - float sumf = 0; - - for (int i = 0; i < nb; ++i) { - - const uint8_t * restrict q4 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const uint16_t * restrict b = (const uint16_t *)x[i].scales; - s16[0] = b[0] & 0x0f0f; - s16[1] = (b[0] >> 4) & 0x0f0f; - - sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3])); - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]); - - size_t vl = 32; - - vint16m1_t vzero = __riscv_vmv_v_x_i16m1(0, 1); - - // load Q4 - vuint8m1_t q4_x = __riscv_vle8_v_u8m1(q4, vl); - - // load Q8 and multiply it with lower Q4 nibble - vint8m1_t q4_a = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vand_vx_u8m1(q4_x, 0x0F, vl)); - vint16m2_t va_0 = __riscv_vwmul_vv_i16m2(q4_a, __riscv_vle8_v_i8m1(q8, vl), vl); - vint16m1_t aux1 = __riscv_vredsum_vs_i16m2_i16m1(va_0, vzero, vl); - - sumf += d*scales[0]*__riscv_vmv_x_s_i16m1_i16(aux1); - - // load Q8 and multiply it with upper Q4 nibble - vint8m1_t q4_s = __riscv_vreinterpret_v_u8m1_i8m1(__riscv_vsrl_vx_u8m1(q4_x, 0x04, vl)); - vint16m2_t va_1 = __riscv_vwmul_vv_i16m2(q4_s, __riscv_vle8_v_i8m1(q8+32, vl), vl); - vint16m1_t aux2 = __riscv_vredsum_vs_i16m2_i16m1(va_1, vzero, vl); - - sumf += d*scales[1]*__riscv_vmv_x_s_i16m1_i16(aux2); - - } - - *s = sumf; - -#elif defined(__POWER9_VECTOR__) - const vector signed char lowMask = vec_splats((signed char)0xF); - const vector unsigned char v4 = vec_splats((unsigned char)0x4); - - vector float vsumf0 = vec_splats(0.0f); - vector float vsumf1 = vec_splats(0.0f); - vector float vsumf2 = vec_splats(0.0f); - vector float vsumf3 = vec_splats(0.0f); - -#pragma GCC unroll 2 - for (int i = 0; i < nb; ++i) { - __builtin_prefetch(x[i].qs, 0, 1); - __builtin_prefetch(y[i].qs, 0, 1); - - vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d[1])); - vector float vyd = vec_splats(y[i].d); - vector float vd= vec_mul(vxd, vyd); - - uint16_t s16[2]; - const uint8_t * scales = (const uint8_t *)s16; - - const uint16_t * restrict b = (const uint16_t *)x[i].scales; - s16[0] = b[0] & 0x0f0f; - s16[1] = (b[0] >> 4) & 0x0f0f; - - vector signed char utmps = (vector signed char)vec_xl_len(scales, 4); - vector signed short vscales = (vector signed short)vec_unpackh(utmps); - vector signed short q4xmins0 = vec_mergeh(vscales, vscales); - q4xmins0 = vec_sld(q4xmins0, q4xmins0, 8); - - vector signed short q8ysums0 = vec_xl_len((const int16_t *)(y[i].bsums), 8); - - vector signed int prod0 = vec_mule(q4xmins0, q8ysums0); - vector signed int prod1 = vec_mulo(q4xmins0, q8ysums0); - - vsumf0 = vec_nmsub(vec_ctf(prod0, 0), vd, vsumf0); - vsumf1 = vec_nmsub(vec_ctf(prod1, 0), vd, vsumf1); - - vd = vec_mul(vyd, vec_splats(GGML_FP16_TO_FP32(x[i].d[0]))); - - vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs); - vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].qs); - vector signed char q4x00 = vec_and(qxs0, lowMask); - vector signed char q4x01 = vec_sr(qxs0, v4); - vector signed char q4x10 = vec_and(qxs1, lowMask); - vector signed char q4x11 = vec_sr(qxs1, v4); - - vector signed char q8y00 = vec_xl( 0, y[i].qs); - vector signed char q8y10 = vec_xl(16, y[i].qs); - vector signed char q8y01 = vec_xl(32, y[i].qs); - vector signed char q8y11 = vec_xl(48, y[i].qs); - - vector signed short qv00 = vec_add(vec_mule(q4x00, q8y00), vec_mulo(q4x00, q8y00)); - vector signed short qv01 = vec_add(vec_mule(q4x01, q8y01), vec_mulo(q4x01, q8y01)); - vector signed short qv10 = vec_add(vec_mule(q4x10, q8y10), vec_mulo(q4x10, q8y10)); - vector signed short qv11 = vec_add(vec_mule(q4x11, q8y11), vec_mulo(q4x11, q8y11)); - - vector signed short vs0 = vec_splat(vscales, 0); - vector signed short vs1 = vec_splat(vscales, 1); - - vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); - vector signed int vsumi1 = vec_add(vec_mule(qv10, vs0), vec_mulo(qv10, vs0)); - vector signed int vsumi2 = vec_add(vec_mule(qv01, vs1), vec_mulo(qv01, vs1)); - vector signed int vsumi3 = vec_add(vec_mule(qv11, vs1), vec_mulo(qv11, vs1)); - - vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); - vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); - vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); - vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); - } - - vsumf0 = vec_add(vsumf0, vsumf2); - vsumf1 = vec_add(vsumf1, vsumf3); - - vsumf0 = vec_add(vsumf0, vsumf1); - - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); - - *s = vec_extract(vsumf0, 0); - -#elif defined __loongarch_asx - - const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); - - __m256 acc = (__m256)__lasx_xvldi(0); - - float summs = 0; - - uint16_t aux16[2]; - const uint8_t * scales = (const uint8_t *)aux16; - - for (int i = 0; i < nb; ++i) { - - const float d = GGML_FP16_TO_FP32(x[i].d[0]) * y[i].d; - const float m = GGML_FP16_TO_FP32(x[i].d[1]) * y[i].d; - const __m256 vd = __lasx_xvreplfr2vr_s(d); - - const uint16_t * a = (const uint16_t *)x[i].scales; - aux16[0] = a[0] & 0x0f0f; - aux16[1] = (a[0] >> 4) & 0x0f0f; - - summs += m * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3])); - - const uint8_t * restrict q4 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); - const __m256i q4l = __lasx_xvand_v(q4bits, m4); - const __m256i q4h = __lasx_xvand_v(__lasx_xvsrli_h(q4bits, 4), m4); - - const __m256i q8l = __lasx_xvld((const __m256i*)(q8+ 0), 0); - const __m256i q8h = __lasx_xvld((const __m256i*)(q8+32), 0); - - const __m256i p16l = lasx_maddubs_h(q4l, q8l); - const __m256i p16h = lasx_maddubs_h(q4h, q8h); - - const __m256i p32l = lasx_madd_h(__lasx_xvreplgr2vr_h(scales[0]), p16l); - acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(p32l), acc); - - const __m256i p32h = lasx_madd_h(__lasx_xvreplgr2vr_h(scales[1]), p16h); - acc = __lasx_xvfmadd_s(vd, __lasx_xvffint_s_w(p32h), acc); - } - - *s = hsum_float_8(acc) - summs; - -#else - - uint8_t aux8[QK_K]; - int16_t aux16[16]; - float sums [8]; - memset(sums, 0, 8*sizeof(float)); - - uint16_t s16[2]; - const uint8_t * restrict scales = (const uint8_t *)s16; - - float sumf = 0; - for (int i = 0; i < nb; ++i) { - const uint8_t * restrict q4 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - uint8_t * restrict a = aux8; - for (int l = 0; l < 32; ++l) a[l+ 0] = q4[l] & 0xF; - for (int l = 0; l < 32; ++l) a[l+32] = q4[l] >> 4; - - const uint16_t * restrict b = (const uint16_t *)x[i].scales; - s16[0] = b[0] & 0x0f0f; - s16[1] = (b[0] >> 4) & 0x0f0f; - - sumf -= y[i].d * GGML_FP16_TO_FP32(x[i].d[1]) * (scales[2] * (y[i].bsums[0] + y[i].bsums[1]) + scales[3] * (y[i].bsums[2] + y[i].bsums[3])); - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d[0]); - - for (int j = 0; j < QK_K/32; ++j) { - for (int l = 0; l < 16; ++l) aux16[l] = q8[l] * a[l]; - q8 += 16; a += 16; - for (int l = 0; l < 16; ++l) aux16[l] += q8[l] * a[l]; - q8 += 16; a += 16; - const float dl = d * scales[j]; - for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[l+8]); - } - } - for (int l = 0; l < 8; ++l) sumf += sums[l]; - *s = sumf; -#endif -} -#endif - -#if QK_K == 256 void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -9151,12 +7554,10 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r float summs = 0.f; - for (int i = 0; i < nb; ++i) { - + for (int i = 0; i < nb; ++i) { const uint8_t * restrict q5 = x[i].qs; const int8_t * restrict q8 = y[i].qs; -#if QK_K == 256 const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); @@ -9166,10 +7567,6 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); utmp[2] = uaux; utmp[0] &= kmask1; -#else - // TODO - const float d = 0, dmin = 0; -#endif const __m256i mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32(utmp[3], utmp[2], utmp[1], utmp[0])); @@ -9552,15 +7949,10 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r const uint8_t * restrict q5 = x[i].qs; const int8_t * restrict q8 = y[i].qs; -#if QK_K == 256 const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); const float dmin = -y[i].d * GGML_FP16_TO_FP32(x[i].dmin); memcpy(utmp, x[i].scales, 12); -#else - // TODO - const float d = 0, dmin = 0; -#endif const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0])); @@ -9679,402 +8071,6 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r #endif } -#else - -void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { - assert(n % QK_K == 0); - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - - const block_q5_K * restrict x = vx; - const block_q8_K * restrict y = vy; - - const int nb = n / QK_K; - -#ifdef __ARM_NEON - const uint8x16_t m4b = vdupq_n_u8(0xf); - const uint8x16_t mh = vdupq_n_u8(16); - const int32x4_t mzero = vdupq_n_s32(0); - - ggml_int8x16x4_t q5bytes; - ggml_uint8x16x4_t q5h; - - float sumf = 0; - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const int8_t * sc = x[i].scales; - - const uint8_t * restrict q5 = x[i].qs; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - - const uint8x8_t qhbits = vld1_u8(qh); - - const ggml_uint8x16x2_t q5bits = ggml_vld1q_u8_x2(q5); - const ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); - - const uint8x16_t htmp = vcombine_u8(qhbits, vshr_n_u8(qhbits, 1)); - q5h.val[0] = vbicq_u8(mh, vshlq_n_u8(htmp, 4)); - q5h.val[1] = vbicq_u8(mh, vshlq_n_u8(htmp, 2)); - q5h.val[2] = vbicq_u8(mh, htmp); - q5h.val[3] = vbicq_u8(mh, vshrq_n_u8(htmp, 2)); - - q5bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[0], m4b)), vreinterpretq_s8_u8(q5h.val[0])); - q5bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vandq_u8(q5bits.val[1], m4b)), vreinterpretq_s8_u8(q5h.val[1])); - q5bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[0], 4)), vreinterpretq_s8_u8(q5h.val[2])); - q5bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(q5bits.val[1], 4)), vreinterpretq_s8_u8(q5h.val[3])); - - int32_t sumi1 = sc[0] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[0], q8bytes.val[0])); - int32_t sumi2 = sc[1] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[1], q8bytes.val[1])); - int32_t sumi3 = sc[2] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[2], q8bytes.val[2])); - int32_t sumi4 = sc[3] * vaddvq_s32(ggml_vdotq_s32(mzero, q5bytes.val[3], q8bytes.val[3])); - - sumf += d * (sumi1 + sumi2 + sumi3 + sumi4); - } - - *s = sumf; - -#elif defined __AVX2__ - - const __m256i m4 = _mm256_set1_epi8(0xF); - const __m256i mone = _mm256_set1_epi8(1); - - __m256 acc = _mm256_setzero_ps(); - - for (int i = 0; i < nb; ++i) { - - const uint8_t * restrict q5 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); - - const __m256i scale_l = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[1]), _mm_set1_epi16(x[i].scales[0])); - const __m256i scale_h = MM256_SET_M128I(_mm_set1_epi16(x[i].scales[3]), _mm_set1_epi16(x[i].scales[2])); - - int64_t aux64; - memcpy(&aux64, x[i].qh, 8); - const __m128i haux128 = _mm_set_epi64x(aux64 >> 1, aux64); - const __m256i haux256 = MM256_SET_M128I(_mm_srli_epi16(haux128, 2), haux128); - - const __m256i q5h_0 = _mm256_slli_epi16(_mm256_andnot_si256(haux256, mone), 4); - const __m256i q5h_1 = _mm256_slli_epi16(_mm256_andnot_si256(_mm256_srli_epi16(haux256, 4), mone), 4); - - const __m256i q5l_0 = _mm256_and_si256(q5bits, m4); - const __m256i q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4); - - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - const __m256i p16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5l_0, q8_0)); - const __m256i p16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5l_1, q8_1)); - const __m256i s16_0 = _mm256_madd_epi16(scale_l, _mm256_maddubs_epi16(q5h_0, q8_0)); - const __m256i s16_1 = _mm256_madd_epi16(scale_h, _mm256_maddubs_epi16(q5h_1, q8_1)); - - const __m256i dot = _mm256_sub_epi32(_mm256_add_epi32(p16_0, p16_1), _mm256_add_epi32(s16_0, s16_1)); - - acc = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(dot), acc); - - } - - *s = hsum_float_8(acc); - -#elif defined __AVX__ - - const __m128i m4 = _mm_set1_epi8(0xF); - const __m128i mone = _mm_set1_epi8(1); - - __m256 acc = _mm256_setzero_ps(); - - for (int i = 0; i < nb; ++i) { - - const uint8_t * restrict q5 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const __m256i q5bits = _mm256_loadu_si256((const __m256i*)q5); - - const __m128i scale_0 = _mm_set1_epi16(x[i].scales[0]); - const __m128i scale_1 = _mm_set1_epi16(x[i].scales[1]); - const __m128i scale_2 = _mm_set1_epi16(x[i].scales[2]); - const __m128i scale_3 = _mm_set1_epi16(x[i].scales[3]); - - int64_t aux64; - memcpy(&aux64, x[i].qh, 8); - const __m128i haux128_0 = _mm_set_epi64x(aux64 >> 1, aux64); - const __m128i haux128_1 = _mm_srli_epi16(haux128_0, 2); - - const __m128i q5h_0 = _mm_slli_epi16(_mm_andnot_si128(haux128_0, mone), 4); - const __m128i q5h_1 = _mm_slli_epi16(_mm_andnot_si128(haux128_1, mone), 4); - const __m128i q5h_2 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_0, 4), mone), 4); - const __m128i q5h_3 = _mm_slli_epi16(_mm_andnot_si128(_mm_srli_epi16(haux128_1, 4), mone), 4); - - const __m128i q5l_0 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 0), m4); - const __m128i q5l_1 = _mm_and_si128(_mm256_extractf128_si256(q5bits, 1), m4); - const __m128i q5l_2 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 0), 4), m4); - const __m128i q5l_3 = _mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q5bits, 1), 4), m4); - - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - const __m128i p16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5l_0, _mm256_extractf128_si256(q8_0, 0))); - const __m128i p16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5l_1, _mm256_extractf128_si256(q8_0, 1))); - const __m128i p16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5l_2, _mm256_extractf128_si256(q8_1, 0))); - const __m128i p16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5l_3, _mm256_extractf128_si256(q8_1, 1))); - const __m128i s16_0 = _mm_madd_epi16(scale_0, _mm_maddubs_epi16(q5h_0, _mm256_extractf128_si256(q8_0, 0))); - const __m128i s16_1 = _mm_madd_epi16(scale_1, _mm_maddubs_epi16(q5h_1, _mm256_extractf128_si256(q8_0, 1))); - const __m128i s16_2 = _mm_madd_epi16(scale_2, _mm_maddubs_epi16(q5h_2, _mm256_extractf128_si256(q8_1, 0))); - const __m128i s16_3 = _mm_madd_epi16(scale_3, _mm_maddubs_epi16(q5h_3, _mm256_extractf128_si256(q8_1, 1))); - - const __m128i dot_0 = _mm_sub_epi32(_mm_add_epi32(p16_0, p16_2), _mm_add_epi32(s16_0, s16_2)); - const __m128i dot_1 = _mm_sub_epi32(_mm_add_epi32(p16_1, p16_3), _mm_add_epi32(s16_1, s16_3)); - - acc = _mm256_add_ps(_mm256_mul_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(MM256_SET_M128I(dot_1, dot_0))), acc); - - } - - *s = hsum_float_8(acc); - -#elif defined __riscv_v_intrinsic - - float sumf = 0; - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const int8_t * sc = x[i].scales; - - const uint8_t * restrict q5 = x[i].qs; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - - vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); - - // load qh - vuint8mf4_t qh_x1 = __riscv_vle8_v_u8mf4(qh, 8); - vuint8mf2_t qh_x2 = __riscv_vlmul_ext_v_u8mf4_u8mf2(__riscv_vsrl_vx_u8mf4(qh_x1, 1, 8)); - - size_t vl = 16; - - // combine both qh_1 and qh_2 - vuint8mf2_t qh_x = __riscv_vslideup_vx_u8mf2(__riscv_vlmul_ext_v_u8mf4_u8mf2(qh_x1), qh_x2, vl/2, vl); - - vuint8mf2_t qh_h0 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl); - vuint8mf2_t qh_h1 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsll_vx_u8mf2(qh_x, 0x2, vl), vl), 16, vl); - vuint8mf2_t qh_h2 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(qh_x, vl), 16, vl); - vuint8mf2_t qh_h3 = __riscv_vand_vx_u8mf2(__riscv_vnot_v_u8mf2(__riscv_vsrl_vx_u8mf2(qh_x, 0x4, vl), vl), 16, vl); - - vint8mf2_t qh_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h0); - vint8mf2_t qh_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h1); - vint8mf2_t qh_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h2); - vint8mf2_t qh_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(qh_h3); - - // load q5 - vuint8mf2_t q5_x1 = __riscv_vle8_v_u8mf2(q5, vl); - vuint8mf2_t q5_x2 = __riscv_vle8_v_u8mf2(q5+16, vl); - - vint8mf2_t q5s_0 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x1, 0xF, vl)); - vint8mf2_t q5s_1 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vand_vx_u8mf2(q5_x2, 0xF, vl)); - vint8mf2_t q5s_2 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x1, 0x4, vl)); - vint8mf2_t q5s_3 = __riscv_vreinterpret_v_u8mf2_i8mf2(__riscv_vsrl_vx_u8mf2(q5_x2, 0x4, vl)); - - vint8mf2_t q5_0 = __riscv_vsub_vv_i8mf2(q5s_0, qh_0, vl); - vint8mf2_t q5_1 = __riscv_vsub_vv_i8mf2(q5s_1, qh_1, vl); - vint8mf2_t q5_2 = __riscv_vsub_vv_i8mf2(q5s_2, qh_2, vl); - vint8mf2_t q5_3 = __riscv_vsub_vv_i8mf2(q5s_3, qh_3, vl); - - // load Q8 and multiply it with Q5 - vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q5_0, __riscv_vle8_v_i8mf2(q8, vl), vl); - vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q5_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl); - vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q5_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl); - vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q5_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl); - - vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl); - vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl); - vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl); - vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl); - - int32_t sumi1 = sc[0] * __riscv_vmv_x_s_i32m1_i32(vs_0); - int32_t sumi2 = sc[1] * __riscv_vmv_x_s_i32m1_i32(vs_1); - int32_t sumi3 = sc[2] * __riscv_vmv_x_s_i32m1_i32(vs_2); - int32_t sumi4 = sc[3] * __riscv_vmv_x_s_i32m1_i32(vs_3); - - sumf += d * (sumi1 + sumi2 + sumi3 + sumi4); - - } - - *s = sumf; - -#elif defined(__POWER9_VECTOR__) - const vector signed char lowMask = vec_splats((signed char)0xF); - const vector unsigned char v1 = vec_splats((unsigned char)0x1); - const vector unsigned char v2 = vec_splats((unsigned char)0x2); - const vector unsigned char v4 = vec_splats((unsigned char)0x4); - - vector float vsumf0 = vec_splats(0.0f); - vector float vsumf1 = vec_splats(0.0f); - vector float vsumf2 = vec_splats(0.0f); - vector float vsumf3 = vec_splats(0.0f); - -#pragma GCC unroll 2 - for (int i = 0; i < nb; ++i) { - __builtin_prefetch(x[i].qs, 0, 1); - __builtin_prefetch(y[i].qs, 0, 1); - - vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); - vector float vyd = vec_splats(y[i].d); - vector float vd= vec_mul(vxd, vyd); - - vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].qs); - vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].qs); - vector signed char qxs00 = (vector signed char)vec_and(qxs0, lowMask); - vector signed char qxs01 = (vector signed char)vec_sr(qxs0, v4); - vector signed char qxs10 = (vector signed char)vec_and(qxs1, lowMask); - vector signed char qxs11 = (vector signed char)vec_sr(qxs1, v4); - - vector signed char qxhs = (vector signed char)vec_xl_len(x[i].qh, 8); - vector signed char qxhs0 = vec_or(qxhs, vec_sr(vec_sld(qxhs, qxhs, 8), v1)); - vector signed char qxhs1 = vec_sr(qxhs0, v2); - vector signed char qxh00 = vec_sl(vec_andc((vector signed char)v1, qxhs0), v4); - vector signed char qxh10 = vec_sl(vec_andc((vector signed char)v1, qxhs1), v4); - vector signed char qxh01 = vec_sl(vec_andc((vector signed char)v1, vec_sr(qxhs0, v4)), v4); - vector signed char qxh11 = vec_sl(vec_andc((vector signed char)v1, vec_sr(qxhs1, v4)), v4); - - vector signed char q5x00 = vec_sub(qxs00, qxh00); - vector signed char q5x10 = vec_sub(qxs10, qxh10); - vector signed char q5x01 = vec_sub(qxs01, qxh01); - vector signed char q5x11 = vec_sub(qxs11, qxh11); - - vector signed char q8y00 = vec_xl( 0, y[i].qs); - vector signed char q8y10 = vec_xl(16, y[i].qs); - vector signed char q8y01 = vec_xl(32, y[i].qs); - vector signed char q8y11 = vec_xl(48, y[i].qs); - - vector signed short qv00 = vec_add(vec_mule(q5x00, q8y00), vec_mulo(q5x00, q8y00)); - vector signed short qv01 = vec_add(vec_mule(q5x01, q8y01), vec_mulo(q5x01, q8y01)); - vector signed short qv10 = vec_add(vec_mule(q5x10, q8y10), vec_mulo(q5x10, q8y10)); - vector signed short qv11 = vec_add(vec_mule(q5x11, q8y11), vec_mulo(q5x11, q8y11)); - - vector signed short vs = (vector signed short)vec_unpackh(vec_xl_len(x[i].scales, 4)); - vector signed short vs0 = vec_splat(vs, 0); - vector signed short vs1 = vec_splat(vs, 1); - vector signed short vs2 = vec_splat(vs, 2); - vector signed short vs3 = vec_splat(vs, 3); - - vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); - vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1)); - vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2)); - vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3)); - - vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); - vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); - vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); - vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); - } - - vsumf0 = vec_add(vsumf0, vsumf2); - vsumf1 = vec_add(vsumf1, vsumf3); - - vsumf0 = vec_add(vsumf0, vsumf1); - - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); - - *s = vec_extract(vsumf0, 0); - -#elif defined __loongarch_asx - - const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); - const __m256i mone = __lasx_xvreplgr2vr_b(1); - - __m256 acc = (__m256)__lasx_xvldi(0); - - for (int i = 0; i < nb; ++i) { - - const uint8_t * restrict q5 = x[i].qs; - const int8_t * restrict q8 = y[i].qs; - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); - - const __m256i scale_l = lasx_insertf128(__lsx_vreplgr2vr_h(x[i].scales[1]), __lsx_vreplgr2vr_h(x[i].scales[0])); - const __m256i scale_h = lasx_insertf128(__lsx_vreplgr2vr_h(x[i].scales[3]), __lsx_vreplgr2vr_h(x[i].scales[2])); - - int64_t aux64; - memcpy(&aux64, x[i].qh, 8); - __m128i haux128 = __lsx_vinsgr2vr_d(haux128, aux64, 0); - haux128 = __lsx_vinsgr2vr_d(haux128, aux64 >> 1, 1); - const __m256i haux256 = lasx_insertf128(__lsx_vsrli_h(haux128, 2), haux128); - - const __m256i q5h_0 = __lasx_xvslli_h(__lasx_xvandn_v(haux256, mone), 4); - const __m256i q5h_1 = __lasx_xvslli_h(__lasx_xvandn_v(__lasx_xvsrli_h(haux256, 4), mone), 4); - - const __m256i q5l_0 = __lasx_xvand_v(q5bits, m4); - const __m256i q5l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q5bits, 4), m4); - - const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0); - const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0); - - const __m256i p16_0 = lasx_madd_h(scale_l, lasx_maddubs_h(q5l_0, q8_0)); - const __m256i p16_1 = lasx_madd_h(scale_h, lasx_maddubs_h(q5l_1, q8_1)); - const __m256i s16_0 = lasx_madd_h(scale_l, lasx_maddubs_h(q5h_0, q8_0)); - const __m256i s16_1 = lasx_madd_h(scale_h, lasx_maddubs_h(q5h_1, q8_1)); - - const __m256i dot = __lasx_xvsub_w(__lasx_xvadd_w(p16_0, p16_1), __lasx_xvadd_w(s16_0, s16_1)); - - acc = __lasx_xvfmadd_s((__m256)__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(dot), acc); - } - - *s = hsum_float_8(acc); - -#else - - int8_t aux8[QK_K]; - int16_t aux16[16]; - float sums [8]; - memset(sums, 0, 8*sizeof(float)); - - float sumf = 0; - for (int i = 0; i < nb; ++i) { - const uint8_t * restrict q4 = x[i].qs; - const uint8_t * restrict hm = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - int8_t * restrict a = aux8; - for (int l = 0; l < 32; ++l) { - a[l+ 0] = q4[l] & 0xF; - a[l+32] = q4[l] >> 4; - } - for (int is = 0; is < 8; ++is) { - uint8_t m = 1 << is; - for (int l = 0; l < 8; ++l) a[8*is + l] -= (hm[l] & m ? 0 : 16); - } - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - const int8_t * restrict sc = x[i].scales; - - for (int j = 0; j < QK_K/16; ++j) { - const float dl = d * sc[j]; - for (int l = 0; l < 16; ++l) aux16[l] = q8[l] * a[l]; - for (int l = 0; l < 8; ++l) sums[l] += dl * (aux16[l] + aux16[8+l]); - q8 += 16; a += 16; - } - } - for (int l = 0; l < 8; ++l) sumf += sums[l]; - *s = sumf; -#endif -} -#endif - - -#if QK_K == 256 void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -10733,446 +8729,6 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r #endif } -#else - -void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * restrict vx, size_t bx, const void * restrict vy, size_t by, int nrc) { - assert(n % QK_K == 0); - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - - const block_q6_K * restrict x = vx; - const block_q8_K * restrict y = vy; - - const int nb = n / QK_K; - -#ifdef __ARM_NEON - float sum = 0; - - const uint8x16_t m4b = vdupq_n_u8(0xF); - const int8x16_t m32s = vdupq_n_s8(32); - const int32x4_t vzero = vdupq_n_s32(0); - - const uint8x16_t mone = vdupq_n_u8(3); - - ggml_int8x16x4_t q6bytes; - ggml_uint8x16x4_t q6h; - - for (int i = 0; i < nb; ++i) { - - const float d_all = GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q6 = x[i].ql; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - - const int8_t * restrict scale = x[i].scales; - - int32_t isum = 0; - - uint8x16_t qhbits = vld1q_u8(qh); - ggml_uint8x16x2_t q6bits = ggml_vld1q_u8_x2(q6); - ggml_int8x16x4_t q8bytes = ggml_vld1q_s8_x4(q8); - - q6h.val[0] = vshlq_n_u8(vandq_u8(mone, qhbits), 4); - uint8x16_t shifted = vshrq_n_u8(qhbits, 2); - q6h.val[1] = vshlq_n_u8(vandq_u8(mone, shifted), 4); - shifted = vshrq_n_u8(qhbits, 4); - q6h.val[2] = vshlq_n_u8(vandq_u8(mone, shifted), 4); - shifted = vshrq_n_u8(qhbits, 6); - q6h.val[3] = vshlq_n_u8(vandq_u8(mone, shifted), 4); - - q6bytes.val[0] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[0], m4b), q6h.val[0])), m32s); - q6bytes.val[1] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.val[1], m4b), q6h.val[1])), m32s); - q6bytes.val[2] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[0], 4), q6h.val[2])), m32s); - q6bytes.val[3] = vsubq_s8(vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.val[1], 4), q6h.val[3])), m32s); - - isum += vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[0], q8bytes.val[0])) * scale[0] + - vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[1], q8bytes.val[1])) * scale[1] + - vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[2], q8bytes.val[2])) * scale[2] + - vaddvq_s32(ggml_vdotq_s32(vzero, q6bytes.val[3], q8bytes.val[3])) * scale[3]; - - sum += isum * d_all * y[i].d; - - } - *s = sum; - -#elif defined __AVX2__ - - const __m256i m4 = _mm256_set1_epi8(0xF); - const __m256i m2 = _mm256_set1_epi8(3); - const __m256i m32s = _mm256_set1_epi8(32); - - __m256 acc = _mm256_setzero_ps(); - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q4 = x[i].ql; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - - const __m64 scales_1 = _mm_set1_pi8(x[i].scales[0]); - const __m64 scales_2 = _mm_set1_pi8(x[i].scales[1]); - const __m64 scales_3 = _mm_set1_pi8(x[i].scales[2]); - const __m64 scales_4 = _mm_set1_pi8(x[i].scales[3]); - - __m256i sumi = _mm256_setzero_si256(); - - const __m128i scale_0 = _mm_set_epi64(scales_2, scales_1); - const __m128i scale_1 = _mm_set_epi64(scales_4, scales_3); - - const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); - const __m128i q4bitsH = _mm_loadu_si128((const __m128i*)qh); - - const __m256i q4h_0 = _mm256_slli_epi16(_mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q4bitsH, 2), q4bitsH), m2), 4); - const __m256i q4h_1 = _mm256_slli_epi16(_mm256_and_si256(MM256_SET_M128I(_mm_srli_epi16(q4bitsH, 6), _mm_srli_epi16(q4bitsH, 4)), m2), 4); - - const __m256i q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0); - const __m256i q4_1 = _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_1); - - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - __m256i q8s_0 = _mm256_maddubs_epi16(m32s, q8_0); - __m256i q8s_1 = _mm256_maddubs_epi16(m32s, q8_1); - - __m256i p16_0 = _mm256_maddubs_epi16(q4_0, q8_0); - __m256i p16_1 = _mm256_maddubs_epi16(q4_1, q8_1); - - p16_0 = _mm256_sub_epi16(p16_0, q8s_0); - p16_1 = _mm256_sub_epi16(p16_1, q8s_1); - - p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0); - p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1); - - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); - - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); - } - - *s = hsum_float_8(acc); - -#elif defined __AVX__ - - const __m128i m4 = _mm_set1_epi8(0xF); - const __m128i m2 = _mm_set1_epi8(3); - const __m128i m32s = _mm_set1_epi8(32); - - __m256 acc = _mm256_setzero_ps(); - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q4 = x[i].ql; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - - const __m64 scales_1 = _mm_set1_pi8(x[i].scales[0]); - const __m64 scales_2 = _mm_set1_pi8(x[i].scales[1]); - const __m64 scales_3 = _mm_set1_pi8(x[i].scales[2]); - const __m64 scales_4 = _mm_set1_pi8(x[i].scales[3]); - - __m128i sumi_0 = _mm_setzero_si128(); - __m128i sumi_1 = _mm_setzero_si128(); - - const __m128i scale_0 = _mm_set_epi64(scales_2, scales_1); - const __m128i scale_1 = _mm_set_epi64(scales_4, scales_3); - - const __m256i q4bits1 = _mm256_loadu_si256((const __m256i*)q4); - const __m128i q4bitsH = _mm_loadu_si128((const __m128i*)qh); - - const __m128i q4h_0 = _mm_slli_epi16(_mm_and_si128(q4bitsH, m2), 4); - const __m128i q4h_1 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH, 2), m2), 4); - const __m128i q4h_2 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH, 4), m2), 4); - const __m128i q4h_3 = _mm_slli_epi16(_mm_and_si128(_mm_srli_epi16(q4bitsH, 6), m2), 4); - - const __m128i q4_0 = _mm_or_si128(_mm_and_si128(_mm256_extractf128_si256(q4bits1, 0), m4), q4h_0); - const __m128i q4_1 = _mm_or_si128(_mm_and_si128(_mm256_extractf128_si256(q4bits1, 1), m4), q4h_1); - const __m128i q4_2 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q4bits1, 0), 4), m4), q4h_2); - const __m128i q4_3 = _mm_or_si128(_mm_and_si128(_mm_srli_epi16(_mm256_extractf128_si256(q4bits1, 1), 4), m4), q4h_3); - - const __m256i q8_0 = _mm256_loadu_si256((const __m256i*)(q8+ 0)); - const __m256i q8_1 = _mm256_loadu_si256((const __m256i*)(q8+32)); - - __m128i q8s_0 = _mm_maddubs_epi16(m32s, _mm256_extractf128_si256(q8_0, 0)); - __m128i q8s_1 = _mm_maddubs_epi16(m32s, _mm256_extractf128_si256(q8_0, 1)); - __m128i q8s_2 = _mm_maddubs_epi16(m32s, _mm256_extractf128_si256(q8_1, 0)); - __m128i q8s_3 = _mm_maddubs_epi16(m32s, _mm256_extractf128_si256(q8_1, 1)); - - __m128i p16_0 = _mm_maddubs_epi16(q4_0, _mm256_extractf128_si256(q8_0, 0)); - __m128i p16_1 = _mm_maddubs_epi16(q4_1, _mm256_extractf128_si256(q8_0, 1)); - __m128i p16_2 = _mm_maddubs_epi16(q4_2, _mm256_extractf128_si256(q8_1, 0)); - __m128i p16_3 = _mm_maddubs_epi16(q4_3, _mm256_extractf128_si256(q8_1, 1)); - - p16_0 = _mm_sub_epi16(p16_0, q8s_0); - p16_1 = _mm_sub_epi16(p16_1, q8s_1); - p16_2 = _mm_sub_epi16(p16_2, q8s_2); - p16_3 = _mm_sub_epi16(p16_3, q8s_3); - - p16_0 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_0), p16_0); - p16_1 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_0, scale_0)), p16_1); - p16_2 = _mm_madd_epi16(_mm_cvtepi8_epi16(scale_1), p16_2); - p16_3 = _mm_madd_epi16(_mm_cvtepi8_epi16(_mm_unpackhi_epi64(scale_1, scale_1)), p16_3); - - sumi_0 = _mm_add_epi32(sumi_0, _mm_add_epi32(p16_0, p16_2)); - sumi_1 = _mm_add_epi32(sumi_1, _mm_add_epi32(p16_1, p16_3)); - - acc = _mm256_add_ps(_mm256_mul_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(MM256_SET_M128I(sumi_1, sumi_0))), acc); - } - - *s = hsum_float_8(acc); - -#elif defined __riscv_v_intrinsic - - float sumf = 0; - - for (int i = 0; i < nb; ++i) { - - const float d_all = GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q6 = x[i].ql; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - - const int8_t * restrict scale = x[i].scales; - - int32_t isum = 0; - - size_t vl = 16; - - vint32m1_t vzero = __riscv_vmv_v_x_i32m1(0, 1); - - // load Q6 - vuint8mf2_t q6_0 = __riscv_vle8_v_u8mf2(q6, vl); - vuint8mf2_t q6_1 = __riscv_vle8_v_u8mf2(q6+16, vl); - - // load qh - vuint8mf2_t qh_x = __riscv_vle8_v_u8mf2(qh, vl); - - vuint8mf2_t qh0 = __riscv_vsll_vx_u8mf2(__riscv_vand_vx_u8mf2(qh_x, 0x3, vl), 0x4, vl); - qh_x = __riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl); - vuint8mf2_t qh1 = __riscv_vsll_vx_u8mf2(__riscv_vand_vx_u8mf2(qh_x, 0x3, vl), 0x4, vl); - qh_x = __riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl); - vuint8mf2_t qh2 = __riscv_vsll_vx_u8mf2(__riscv_vand_vx_u8mf2(qh_x, 0x3, vl), 0x4, vl); - qh_x = __riscv_vsrl_vx_u8mf2(qh_x, 0x2, vl); - vuint8mf2_t qh3 = __riscv_vsll_vx_u8mf2(__riscv_vand_vx_u8mf2(qh_x, 0x3, vl), 0x4, vl); - - vuint8mf2_t q6h_0 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(q6_0, 0xF, vl), qh0, vl); - vuint8mf2_t q6h_1 = __riscv_vor_vv_u8mf2(__riscv_vand_vx_u8mf2(q6_1, 0xF, vl), qh1, vl); - vuint8mf2_t q6h_2 = __riscv_vor_vv_u8mf2(__riscv_vsrl_vx_u8mf2(q6_0, 0x4, vl), qh2, vl); - vuint8mf2_t q6h_3 = __riscv_vor_vv_u8mf2(__riscv_vsrl_vx_u8mf2(q6_1, 0x4, vl), qh3, vl); - - vint8mf2_t q6v_0 = __riscv_vsub_vx_i8mf2(__riscv_vreinterpret_v_u8mf2_i8mf2(q6h_0), 32, vl); - vint8mf2_t q6v_1 = __riscv_vsub_vx_i8mf2(__riscv_vreinterpret_v_u8mf2_i8mf2(q6h_1), 32, vl); - vint8mf2_t q6v_2 = __riscv_vsub_vx_i8mf2(__riscv_vreinterpret_v_u8mf2_i8mf2(q6h_2), 32, vl); - vint8mf2_t q6v_3 = __riscv_vsub_vx_i8mf2(__riscv_vreinterpret_v_u8mf2_i8mf2(q6h_3), 32, vl); - - // load Q8 and take product - vint16m1_t p0 = __riscv_vwmul_vv_i16m1(q6v_0, __riscv_vle8_v_i8mf2(q8, vl), vl); - vint16m1_t p1 = __riscv_vwmul_vv_i16m1(q6v_1, __riscv_vle8_v_i8mf2(q8+16, vl), vl); - vint16m1_t p2 = __riscv_vwmul_vv_i16m1(q6v_2, __riscv_vle8_v_i8mf2(q8+32, vl), vl); - vint16m1_t p3 = __riscv_vwmul_vv_i16m1(q6v_3, __riscv_vle8_v_i8mf2(q8+48, vl), vl); - - vint32m1_t vs_0 = __riscv_vwredsum_vs_i16m1_i32m1(p0, vzero, vl); - vint32m1_t vs_1 = __riscv_vwredsum_vs_i16m1_i32m1(p1, vzero, vl); - vint32m1_t vs_2 = __riscv_vwredsum_vs_i16m1_i32m1(p2, vzero, vl); - vint32m1_t vs_3 = __riscv_vwredsum_vs_i16m1_i32m1(p3, vzero, vl); - - isum += __riscv_vmv_x_s_i32m1_i32(vs_0) * scale[0]; - isum += __riscv_vmv_x_s_i32m1_i32(vs_1) * scale[1]; - isum += __riscv_vmv_x_s_i32m1_i32(vs_2) * scale[2]; - isum += __riscv_vmv_x_s_i32m1_i32(vs_3) * scale[3]; - - sumf += isum * d_all * y[i].d; - - } - - *s = sumf; - -#elif defined(__POWER9_VECTOR__) - const vector signed char lowMask = vec_splats((signed char)0xF); - const vector unsigned char v2 = vec_splats((unsigned char)0x2); - const vector unsigned char v3 = vec_splats((unsigned char)0x3); - const vector unsigned char v4 = vec_splats((unsigned char)0x4); - const vector unsigned char v6 = vec_splats((unsigned char)0x6); - const vector signed char off = vec_splats((signed char)0x20); - - vector float vsumf0 = vec_splats(0.0f); - vector float vsumf1 = vec_splats(0.0f); - vector float vsumf2 = vec_splats(0.0f); - vector float vsumf3 = vec_splats(0.0f); - -#pragma GCC unroll 2 - for (int i = 0; i < nb; ++i) { - __builtin_prefetch(x[i].ql, 0, 1); - __builtin_prefetch(x[i].qh, 0, 1); - __builtin_prefetch(y[i].qs, 0, 1); - - vector float vxd = vec_splats(GGML_FP16_TO_FP32(x[i].d)); - vector float vyd = vec_splats(y[i].d); - vector float vd= vec_mul(vxd, vyd); - - vector signed char qxs0 = (vector signed char)vec_xl( 0, x[i].ql); - vector signed char qxs1 = (vector signed char)vec_xl(16, x[i].ql); - vector signed char qxs00 = vec_and(qxs0, lowMask); - vector signed char qxs01 = vec_sr(qxs0, v4); - vector signed char qxs10 = vec_and(qxs1, lowMask); - vector signed char qxs11 = vec_sr(qxs1, v4); - - vector signed char qxhs0 = (vector signed char)vec_xl( 0, x[i].qh); - - vector signed char qxh00 = vec_sl(vec_and((vector signed char)v3, qxhs0), v4); - vector signed char qxh01 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v4)), v4); - vector signed char qxh10 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v2)), v4); - vector signed char qxh11 = vec_sl(vec_and((vector signed char)v3, vec_sr(qxhs0, v6)), v4); - - vector signed char q6x00 = vec_sub(vec_or(qxh00, qxs00), off); - vector signed char q6x01 = vec_sub(vec_or(qxh01, qxs01), off); - vector signed char q6x10 = vec_sub(vec_or(qxh10, qxs10), off); - vector signed char q6x11 = vec_sub(vec_or(qxh11, qxs11), off); - - vector signed char q8y00 = vec_xl( 0, y[i].qs); - vector signed char q8y10 = vec_xl(16, y[i].qs); - vector signed char q8y01 = vec_xl(32, y[i].qs); - vector signed char q8y11 = vec_xl(48, y[i].qs); - - vector signed short qv00 = vec_add(vec_mule(q6x00, q8y00), vec_mulo(q6x00, q8y00)); - vector signed short qv10 = vec_add(vec_mule(q6x10, q8y10), vec_mulo(q6x10, q8y10)); - vector signed short qv01 = vec_add(vec_mule(q6x01, q8y01), vec_mulo(q6x01, q8y01)); - vector signed short qv11 = vec_add(vec_mule(q6x11, q8y11), vec_mulo(q6x11, q8y11)); - - vector signed short vs = (vector signed short)vec_unpackh(vec_xl_len(x[i].scales, 4)); - vector signed short vs0 = vec_splat(vs, 0); - vector signed short vs1 = vec_splat(vs, 1); - vector signed short vs2 = vec_splat(vs, 2); - vector signed short vs3 = vec_splat(vs, 3); - - vector signed int vsumi0 = vec_add(vec_mule(qv00, vs0), vec_mulo(qv00, vs0)); - vector signed int vsumi1 = vec_add(vec_mule(qv10, vs1), vec_mulo(qv10, vs1)); - vector signed int vsumi2 = vec_add(vec_mule(qv01, vs2), vec_mulo(qv01, vs2)); - vector signed int vsumi3 = vec_add(vec_mule(qv11, vs3), vec_mulo(qv11, vs3)); - - vsumf0 = vec_madd(vec_ctf(vsumi0, 0), vd, vsumf0); - vsumf1 = vec_madd(vec_ctf(vsumi1, 0), vd, vsumf1); - vsumf2 = vec_madd(vec_ctf(vsumi2, 0), vd, vsumf2); - vsumf3 = vec_madd(vec_ctf(vsumi3, 0), vd, vsumf3); - } - - vsumf0 = vec_add(vsumf0, vsumf2); - vsumf1 = vec_add(vsumf1, vsumf3); - - vsumf0 = vec_add(vsumf0, vsumf1); - - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 4)); - vsumf0 = vec_add(vsumf0, vec_sld(vsumf0, vsumf0, 8)); - - *s = vec_extract(vsumf0, 0); - -#elif defined __loongarch_asx - - const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); - const __m256i m2 = __lasx_xvreplgr2vr_b(3); - const __m256i m32s = __lasx_xvreplgr2vr_b(32); - - __m256 acc = (__m256)__lasx_xvldi(0); - - for (int i = 0; i < nb; ++i) { - - const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); - - const uint8_t * restrict q4 = x[i].ql; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - - const __m64 scales_1 = __lasx_xvreplgr2vr_b(x[i].scales[0]); - const __m64 scales_2 = __lasx_xvreplgr2vr_b(x[i].scales[1]); - const __m64 scales_3 = __lasx_xvreplgr2vr_b(x[i].scales[2]); - const __m64 scales_4 = __lasx_xvreplgr2vr_b(x[i].scales[3]); - - __m256i sumi = __lasx_xvldi(0); - - __m128i scale_0 = __lsx_vinsgr2vr_d(scale_0, scales_1, 0); - scale_0 = __lsx_vinsgr2vr_d(scale_0, scales_2, 1); - __m128i scale_1 = __lsx_vinsgr2vr_d(scale_1, scales_3, 0); - scale_1 = __lsx_vinsgr2vr_d(scale_1, scales_4, 1); - - const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); - const __m128i q4bitsH = __lsx_vld((const __m128i*)qh, 0); - - const __m256i q4h_0 = __lasx_xvslli_h(__lasx_xvand_v(lasx_insertf128(__lasx_xvsrli_h(q4bitsH, 2), q4bitsH), m2), 4); - const __m256i q4h_1 = __lasx_xvslli_h(__lasx_xvand_v(lasx_insertf128(__lasx_xvsrli_h(q4bitsH, 6), __lasx_xvsrli_h(q4bitsH, 4)), m2), 4); - - const __m256i q4_0 = __lasx_xvor_v(__lasx_xvand_v(q4bits1, m4), q4h_0); - const __m256i q4_1 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits1, 4), m4), q4h_1); - - const __m256i q8_0 = __lasx_xvld((const __m256i*)(q8+ 0), 0); - const __m256i q8_1 = __lasx_xvld((const __m256i*)(q8+32), 0); - - __m256i q8s_0 = lasx_maddubs_h(m32s, q8_0); - __m256i q8s_1 = lasx_maddubs_h(m32s, q8_1); - - __m256i p16_0 = lasx_maddubs_h(q4_0, q8_0); - __m256i p16_1 = lasx_maddubs_h(q4_1, q8_1); - - p16_0 = __lasx_xvsub_h(p16_0, q8s_0); - p16_1 = __lasx_xvsub_h(p16_1, q8s_1); - - p16_0 = lasx_madd_h(lasx_ext8_16(scale_0), p16_0); - p16_1 = lasx_madd_h(lasx_ext8_16(scale_1), p16_1); - - sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1)); - - acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc); - } - - *s = hsum_float_8(acc); - -#else - - int8_t aux8[QK_K]; - int16_t aux16[8]; - float sums [8]; - int32_t aux32[8]; - memset(sums, 0, 8*sizeof(float)); - - float sumf = 0; - for (int i = 0; i < nb; ++i) { - const uint8_t * restrict q4 = x[i].ql; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; - memset(aux32, 0, 8*sizeof(int32_t)); - int8_t * restrict a = aux8; - for (int l = 0; l < 16; ++l) { - a[l+ 0] = (int8_t)((q4[l+ 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32; - a[l+16] = (int8_t)((q4[l+16] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32; - a[l+32] = (int8_t)((q4[l+ 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32; - a[l+48] = (int8_t)((q4[l+16] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32; - } - int is = 0; - for (int j = 0; j < QK_K/16; ++j) { - int scale = x[i].scales[is++]; - for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l]; - for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l]; - q8 += 8; a += 8; - for (int l = 0; l < 8; ++l) aux16[l] = q8[l] * a[l]; - for (int l = 0; l < 8; ++l) aux32[l] += scale * aux16[l]; - q8 += 8; a += 8; - } - const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; - for (int l = 0; l < 8; ++l) sums[l] += d * aux32[l]; - } - for (int l = 0; l < 8; ++l) sumf += sums[l]; - *s = sumf; -#endif -} - -#endif - #if defined (__AVX2__) || defined (__ARM_NEON) || defined (__POWER9_VECTOR__) || defined(__loongarch_asx) static const int8_t keven_signs_q2xs[1024] = { 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, 1, 1, @@ -11564,64 +9120,6 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * const __m256i block_sign_shuffle_1 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_1); const __m256i block_sign_shuffle_2 = _mm256_loadu_si256((const __m256i*)block_sign_shuffle_mask_2); -#if QK_K == 64 - static const uint8_t k_bit_helper[16] = { - 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, - }; - const __m128i bit_helper = _mm_loadu_si128((const __m128i*)k_bit_helper); - const __m128i m511 = _mm_set1_epi16(511); - typedef union { - __m128i vec_index; - uint16_t index[8]; - } index_t; - - index_t idx; - __m256 accumf = _mm256_setzero_ps(); - for (int i = 0; i < nb; ++i) { - const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; - const __m128i q2_data = _mm_loadu_si128((const __m128i*)x[i].qs); - idx.vec_index = _mm_and_si128(q2_data, m511); - - const __m128i partial_sign_bits = _mm_srli_epi16(q2_data, 9); - const __m128i partial_sign_bits_upper = _mm_srli_epi16(q2_data, 13); - const __m128i partial_sign_bits_for_counting = _mm_xor_si128(partial_sign_bits, partial_sign_bits_upper); - - const __m128i odd_bits = _mm_shuffle_epi8(bit_helper, partial_sign_bits_for_counting); - const __m128i full_sign_bits = _mm_or_si128(partial_sign_bits, odd_bits); - const __m256i full_signs = MM256_SET_M128I(full_sign_bits, full_sign_bits); - - const __m256i q8_1 = _mm256_loadu_si256((const __m256i *)y[i].qs); - const __m256i q8_2 = _mm256_loadu_si256((const __m256i *)(y[i].qs+32)); - - const __m256i q2_1 = _mm256_set_epi64x(iq2xs_grid[idx.index[3]], iq2xs_grid[idx.index[2]], - iq2xs_grid[idx.index[1]], iq2xs_grid[idx.index[0]]); - const __m256i q2_2 = _mm256_set_epi64x(iq2xs_grid[idx.index[7]], iq2xs_grid[idx.index[6]], - iq2xs_grid[idx.index[5]], iq2xs_grid[idx.index[4]]); - - __m256i signs; - signs = _mm256_shuffle_epi8(full_signs, block_sign_shuffle_1); - signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask); - const __m256i q8s_1 = _mm256_sign_epi8(q8_1, _mm256_or_si256(signs, mone)); - - signs = _mm256_shuffle_epi8(full_signs, block_sign_shuffle_2); - signs = _mm256_cmpeq_epi8(_mm256_and_si256(signs, bit_selector_mask), bit_selector_mask); - const __m256i q8s_2 = _mm256_sign_epi8(q8_2, _mm256_or_si256(signs, mone)); - - const __m256i dot1 = _mm256_maddubs_epi16(q2_1, q8s_1); - const __m256i dot2 = _mm256_maddubs_epi16(q2_2, q8s_2); - - const __m256i sc1 = MM256_SET_M128I(_mm_set1_epi16(2*(x[i].scales[0] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[0] & 0xf)+1)); - const __m256i sc2 = MM256_SET_M128I(_mm_set1_epi16(2*(x[i].scales[1] >> 4)+1), _mm_set1_epi16(2*(x[i].scales[1] & 0xf)+1)); - - const __m256i sum = _mm256_add_epi32(_mm256_madd_epi16(sc1, dot1), _mm256_madd_epi16(sc2, dot2)); - - accumf = _mm256_fmadd_ps(_mm256_set1_ps(d), _mm256_cvtepi32_ps(sum), accumf); - - } - - *s = 0.125f * hsum_float_8(accumf); -#else - static const uint8_t k_bit_helper[32] = { 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, @@ -11719,7 +9217,6 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * } *s = 0.125f * hsum_float_8(accumf); -#endif #elif defined(__loongarch_asx) const __m256i mone = __lasx_xvreplgr2vr_b(1); @@ -11740,62 +9237,6 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * const __m256i block_sign_shuffle_1 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_1, 0); const __m256i block_sign_shuffle_2 = __lasx_xvld((const __m256i*)block_sign_shuffle_mask_2, 0); -#if QK_K == 64 - static const uint8_t k_bit_helper[16] = { - 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, - }; - const __m128i bit_helper = __lsx_vld((const __m128i*)k_bit_helper, 0); - const __m128i m511 = __lsx_vreplgr2vr_h(511); - typedef union { - __m128i vec_index; - uint16_t index[8]; - } index_t; - - index_t idx; - __m256 accumf = (__m256)__lasx_xvldi(0); - for (int i = 0; i < nb; ++i) { - const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; - const __m128i q2_data = __lsx_vld((const __m128i*)x[i].qs, 0); - idx.vec_index = __lsx_vand_v(q2_data, m511); - - const __m128i partial_sign_bits = __lsx_vsrli_h(q2_data, 9); - const __m128i partial_sign_bits_upper = __lsx_vsrli_h(q2_data, 13); - const __m128i partial_sign_bits_for_counting = __lsx_vxor_v(partial_sign_bits, partial_sign_bits_upper); - - const __m128i odd_bits = lsx_shuffle_b(bit_helper, partial_sign_bits_for_counting); - const __m128i full_sign_bits = __lsx_vor_v(partial_sign_bits, odd_bits); - const __m256i full_signs = lasx_insertf128(full_sign_bits, full_sign_bits); - - const __m256i q8_1 = __lasx_xvld((const __m256i *)y[i].qs, 0); - const __m256i q8_2 = __lasx_xvld((const __m256i *)(y[i].qs+32), 0); - - const __m256i q2_1 = lasx_set_d(iq2xs_grid[idx.index[3]], iq2xs_grid[idx.index[2]], - iq2xs_grid[idx.index[1]], iq2xs_grid[idx.index[0]]); - const __m256i q2_2 = lasx_set_d(iq2xs_grid[idx.index[7]], iq2xs_grid[idx.index[6]], - iq2xs_grid[idx.index[5]], iq2xs_grid[idx.index[4]]); - __m256i signs; - signs = lasx_shuffle_b(full_signs, block_sign_shuffle_1); - signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask); - const __m256i q8s_1 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_1); - - signs = lasx_shuffle_b(full_signs, block_sign_shuffle_2); - signs = __lasx_xvseq_b(__lasx_xvand_v(signs, bit_selector_mask), bit_selector_mask); - const __m256i q8s_2 = __lasx_xvsigncov_b(__lasx_xvor_v(signs, mone), q8_2); - - const __m256i dot1 = lasx_maddubs_h(q2_1, q8s_1); - const __m256i dot2 = lasx_maddubs_h(q2_2, q8s_2); - - const __m256i sc1 = lasx_insertf128(_mm_set1_epi16(2*(x[i].scales[0] >> 4)+1), __lsx_vreplgr2vr_h(2*(x[i].scales[0] & 0xf)+1)); - const __m256i sc2 = lasx_insertf128(_mm_set1_epi16(2*(x[i].scales[1] >> 4)+1), __lsx_vreplgr2vr_h(2*(x[i].scales[1] & 0xf)+1)); - - const __m256i sum = __lasx_xvadd_w(lasx_madd_h(sc1, dot1), lasx_madd_h(sc2, dot2)); - - accumf = __lasx_vfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sum), accumf); - } - - *s = 0.125f * hsum_float_8(accumf); -#else - static const uint8_t k_bit_helper[32] = { 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00, 0x80, 0x00, 0x80, 0x80, 0x00, @@ -11893,9 +9334,6 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * restrict s, size_t bs, const void * } *s = 0.125f * hsum_float_8(accumf); -#endif - - #elif defined(__POWER9_VECTOR__) vector float vsumf0 = vec_splats(0.0f); vector float vsumf1 = vec_splats(0.0f); @@ -12748,10 +10186,8 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * ggml_int8x16x4_t q8b; vec_index_t idx; -#if QK_K == 256 uint32_t scales32[2]; const uint8_t * scales8 = (const uint8_t *)scales32; -#endif float sumf = 0; for (int i = 0; i < nb; ++i) { @@ -12761,11 +10197,9 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * const uint16_t * restrict signs = (const uint16_t *)x[i].signs; const int8_t * restrict q8 = y[i].qs; -#if QK_K == 256 memcpy(scales32, x[i].scales, 4); scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101; scales32[0] = ((scales32[0] & 0x0f0f0f0f) << 1) | 0x01010101; -#endif int sumi1 = 0, sumi2 = 0; for (int ib32 = 0; ib32 < QK_K/32; ib32 += 2) { @@ -12806,13 +10240,9 @@ void ggml_vec_dot_iq3_s_q8_K (int n, float * restrict s, size_t bs, const void * const int32x4_t p1 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[0], q8b.val[0]), q3s.val[1], q8b.val[1]); const int32x4_t p2 = ggml_vdotq_s32(ggml_vdotq_s32(vdupq_n_s32(0), q3s.val[2], q8b.val[2]), q3s.val[3], q8b.val[3]); -#if QK_K == 256 + sumi1 += vaddvq_s32(p1) * scales8[ib32/2+0]; sumi2 += vaddvq_s32(p2) * scales8[ib32/2+4]; -#else - sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32/2] & 0xf)); - sumi2 += vaddvq_s32(p2) * (1 + 2*(x[i].scales[ib32/2] >> 4)); -#endif } sumf += d*(sumi1 + sumi2); } @@ -13476,17 +10906,10 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void const int nb = n / QK_K; -#if QK_K != 64 iq1m_scale_t scale; -#endif #if defined __ARM_NEON - -#if QK_K == 64 - const int32x4_t mask = vdupq_n_s32(0xf); -#else const int32x4_t mask = vdupq_n_s32(0x7); -#endif const int32x4_t mone = vdupq_n_s32(1); const int32x4_t mzero = vdupq_n_s32(0); @@ -13510,9 +10933,7 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void const uint8_t * qh = x[i].qh; const uint16_t * sc = (const uint16_t *)x[i].scales; -#if QK_K != 64 scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); -#endif int32x4_t sumi1 = mzero; int32x4_t sumi2 = mzero; @@ -13541,11 +10962,8 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void const int32x4_t p4 = vpaddq_s32(ggml_vdotq_s32(mzero, deltas.val[aux8[2]], q8b.val[2]), ggml_vdotq_s32(mzero, deltas.val[aux8[3]], q8b.val[3])); const int32x4_t p34 = vpaddq_s32(p3, p4); -#if QK_K == 64 - int32x4_t scales_4 = ggml_vld1q_u32(sc[0] >> 0, sc[0] >> 4, sc[0] >> 8, sc[0] >> 12); -#else int32x4_t scales_4 = ggml_vld1q_u32(sc[ib/2] >> 0, sc[ib/2] >> 3, sc[ib/2] >> 6, sc[ib/2] >> 9); -#endif + scales_4 = vaddq_s32(vshlq_n_s32(vandq_s32(scales_4, mask), 1), mone); sumi1 = vmlaq_s32(sumi1, scales_4, p12); @@ -13555,22 +10973,14 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void } -#if QK_K == 64 - sumf += y[i].d * GGML_FP16_TO_FP32(x[i].d) * (vaddvq_s32(sumi1) + IQ1M_DELTA * vaddvq_s32(sumi2)); -#else sumf += y[i].d * GGML_FP16_TO_FP32(scale.f16) * (vaddvq_s32(sumi1) + IQ1M_DELTA * vaddvq_s32(sumi2)); -#endif } *s = sumf; #elif defined __AVX2__ -#if QK_K == 64 - const __m256i mask = _mm256_set1_epi16(0xf); -#else const __m256i mask = _mm256_set1_epi16(0x7); -#endif const __m256i mone = _mm256_set1_epi16(1); __m256 accum1 = _mm256_setzero_ps(); @@ -13582,9 +10992,7 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void const uint8_t * qh = x[i].qh; const uint16_t * sc = (const uint16_t *)x[i].scales; -#if QK_K != 64 scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); -#endif __m256i sumi1 = _mm256_setzero_si256(); __m256i sumi2 = _mm256_setzero_si256(); @@ -13614,13 +11022,10 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void const __m256i dot3 = mul_add_epi8(delta1, q8b_1); const __m256i dot4 = mul_add_epi8(delta2, q8b_2); -#if QK_K == 64 - __m256i scale1 = MM256_SET_M128I(_mm_set1_epi16(sc[0] >> 4), _mm_set1_epi16(sc[0] >> 0)); - __m256i scale2 = MM256_SET_M128I(_mm_set1_epi16(sc[0] >> 12), _mm_set1_epi16(sc[0] >> 8)); -#else + __m256i scale1 = MM256_SET_M128I(_mm_set1_epi16(sc[ib/2] >> 3), _mm_set1_epi16(sc[ib/2] >> 0)); __m256i scale2 = MM256_SET_M128I(_mm_set1_epi16(sc[ib/2] >> 9), _mm_set1_epi16(sc[ib/2] >> 6)); -#endif + scale1 = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scale1, mask), 1), mone); scale2 = _mm256_add_epi16(_mm256_slli_epi16(_mm256_and_si256(scale2, mask), 1), mone); const __m256i p1 = _mm256_madd_epi16(dot1, scale1); @@ -13634,14 +11039,10 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void qs += 8; qh += 4; } -#if QK_K == 64 - const __m256 d = _mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(x[i].d)); -#else const __m256 d = _mm256_set1_ps(y[i].d * GGML_FP16_TO_FP32(scale.f16)); -#endif + accum1 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi1), accum1); accum2 = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi2), accum2); - } *s = hsum_float_8(accum1) + IQ1M_DELTA * hsum_float_8(accum2); @@ -13658,9 +11059,7 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void const uint8_t * qh = x[i].qh; const uint16_t * sc = (const uint16_t *)x[i].scales; -#if QK_K != 64 scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); -#endif int sumi1 = 0, sumi2 = 0; for (int ib = 0; ib < QK_K/32; ++ib) { @@ -13680,24 +11079,17 @@ void ggml_vec_dot_iq1_m_q8_K (int n, float * restrict s, size_t bs, const void sum1[l/2] += lsum1; sum2[l/2] += lsum2*delta[l]; } -#if QK_K == 64 - const int ls1 = 2*((sc[0] >> (8*(ib%2)+0)) & 0xf) + 1; - const int ls2 = 2*((sc[0] >> (8*(ib%2)+4)) & 0xf) + 1; -#else + const int ls1 = 2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1; const int ls2 = 2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1; -#endif + sumi1 += sum1[0] * ls1 + sum1[1] * ls2; sumi2 += sum2[0] * ls1 + sum2[1] * ls2; qs += 4; qh += 2; } -#if QK_K == 64 - sumf += GGML_FP16_TO_FP32(x[i].d) * y[i].d * (sumi1 + IQ1M_DELTA * sumi2); -#else sumf += GGML_FP16_TO_FP32(scale.f16) * y[i].d * (sumi1 + IQ1M_DELTA * sumi2); -#endif } *s = sumf; @@ -13885,9 +11277,6 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * UNUSED(by); UNUSED(bs); assert(n % QK_K == 0); -#if QK_K == 64 - ggml_vec_dot_iq4_nl_q8_0(n, s, bs, vx, bx, vy, by, nrc); -#else const block_iq4_xs * restrict x = vx; const block_q8_K * restrict y = vy; @@ -14180,7 +11569,6 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * } *s = sumf; #endif -#endif } // ================================ IQ2 quantization ============================================= @@ -15998,10 +13386,6 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy const float * xx; for (int ibl = 0; ibl < nbl; ++ibl) { - -#if QK_K == 64 - y[ibl].d = GGML_FP32_TO_FP16(0.f); -#endif memset(y[ibl].qs, 0, QK_K/8); memset(y[ibl].qh, 0, QK_K/16); memset(y[ibl].scales, 0, QK_K/32); @@ -16176,22 +13560,13 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy } uint16_t * sc = (uint16_t *)y[ibl].scales; -#if QK_K == 64 - float d = max_scale/31; -#else float d = max_scale/15; -#endif float id = 1/d; float sumqx_f = 0, sumq2_f = 0; for (int ib = 0; ib < QK_K/block_size; ++ib) { int l = nearest_int(0.5f*(id*scales[ib+0]-1)); -#if QK_K == 64 - l = MAX(0, MIN(15, l)); - sc[ib/4] |= (l << 4*(ib%4)); -#else l = MAX(0, MIN(7, l)); sc[ib/4] |= (l << 3*(ib%4)); -#endif y[ibl].qh[ib] |= masks[shifts[ib]]; const float * xb = xbl + block_size*ib; if (quant_weights) { @@ -16214,14 +13589,10 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy } if (sumq2_f > 0) d = sumqx_f/sumq2_f; s.f16 = GGML_FP32_TO_FP16(d*1.1125f); // 1.1125f is another fudge factor. Don't ask me why it is needed. -#if QK_K == 64 - y[ibl].d = s.f16; -#else sc[0] |= ((s.u16 & 0x000f) << 12); sc[1] |= ((s.u16 & 0x00f0) << 8); sc[2] |= ((s.u16 & 0x0f00) << 4); sc[3] |= ((s.u16 & 0xf000) << 0); -#endif } } @@ -16410,9 +13781,6 @@ void quantize_row_iq4_nl_reference(const float * restrict x, block_iq4_nl * rest } size_t quantize_iq4_xs(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { -#if QK_K == 64 - return quantize_iq4_nl(src, dst, nrow, n_per_row, quant_weights); -#else GGML_ASSERT(n_per_row%QK_K == 0); int64_t nblock = n_per_row/QK_K; char * qrow = (char *)dst; @@ -16430,7 +13798,6 @@ size_t quantize_iq4_xs(const float * restrict src, void * restrict dst, int64_t qrow += nblock*sizeof(block_iq4_xs); } return nrow * nblock * sizeof(block_iq4_xs); -#endif } void quantize_row_iq4_xs(const float * restrict x, void * restrict vy, int64_t k) { @@ -16842,19 +14209,11 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte } break; case GGML_TYPE_Q4_K: { - #ifdef GGML_QKK_64 - VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d[0], d[1]); - #else VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d, dmin); - #endif } break; case GGML_TYPE_Q5_K: { - #ifdef GGML_QKK_64 - VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_K, data, nb); - #else VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_K, data, nb, d, dmin); - #endif } break; case GGML_TYPE_Q6_K: { @@ -16877,18 +14236,12 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { const block_iq1_m * q = (const block_iq1_m *) data; for (size_t i = 0; i < nb; ++i) { - #if QK_K == 64 - if (!validate_fp16(q[i].d, i)) { - return false; - } - #else iq1m_scale_t scale; const uint16_t * sc = (const uint16_t *)q[i].scales; scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); if (!validate_fp16(scale.f16, i)) { return false; } - #endif } } break; case GGML_TYPE_IQ2_XXS: @@ -16913,12 +14266,9 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_s, data, nb); } break; case GGML_TYPE_IQ4_XS: - #if QK_K != 64 { VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_xs, data, nb); } break; - #endif - // with QK_K == 64, iq4_xs is iq4_nl case GGML_TYPE_IQ4_NL: { VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb); diff --git a/ggml-sycl.cpp b/ggml-sycl.cpp index f486b6c0a..496ec61c3 100644 --- a/ggml-sycl.cpp +++ b/ggml-sycl.cpp @@ -4197,7 +4197,6 @@ static void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restri const block_q2_K * x = (const block_q2_K *) vx; const int tid = item_ct1.get_local_id(2); -#if QK_K == 256 const int n = tid/32; const int l = tid - 32*n; const int is = 8*n + l/16; @@ -4211,18 +4210,6 @@ static void dequantize_block_q2_K(const void * __restrict__ vx, dst_t * __restri y[l+32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 2) & 3) - dmin * (x[i].scales[is+2] >> 4); y[l+64] = dall * (x[i].scales[is+4] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+4] >> 4); y[l+96] = dall * (x[i].scales[is+6] & 0xF) * ((q >> 6) & 3) - dmin * (x[i].scales[is+6] >> 4); -#else - const int is = tid/16; // 0 or 1 - const int il = tid%16; // 0...15 - const uint8_t q = x[i].qs[il] >> (2*is); - dst_t * y = yy + i*QK_K + 16*is + il; - - float dall = x[i].dm[0]; - float dmin = x[i].dm[1]; - y[ 0] = dall * (x[i].scales[is+0] & 0xF) * ((q >> 0) & 3) - dmin * (x[i].scales[is+0] >> 4); - y[32] = dall * (x[i].scales[is+2] & 0xF) * ((q >> 4) & 3) - dmin * (x[i].scales[is+2] >> 4); -#endif - } template @@ -4232,7 +4219,6 @@ static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restri const int i = item_ct1.get_group(2); const block_q3_K * x = (const block_q3_K *) vx; -#if QK_K == 256 const int r = item_ct1.get_local_id(2) / 4; const int tid = r/2; const int is0 = r%2; @@ -4256,31 +4242,8 @@ static void dequantize_block_q3_K(const void * __restrict__ vx, dst_t * __restri const uint8_t * hm = x[i].hmask; for (int l = l0; l < l0+4; ++l) y[l] = dl * ((int8_t)((q[l] >> shift) & 3) - ((hm[l] & m) ? 0 : 4)); -#else - const int tid = item_ct1.get_local_id(2); - const int is = tid/16; // 0 or 1 - const int il = tid%16; // 0...15 - const int im = il/8; // 0...1 - const int in = il%8; // 0...7 - - dst_t * y = yy + i*QK_K + 16*is + il; - - const uint8_t q = x[i].qs[il] >> (2*is); - const uint8_t h = x[i].hmask[in] >> (2*is + im); - const float d = (float)x[i].d; - - if (is == 0) { - y[ 0] = d * ((x[i].scales[0] & 0xF) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4)); - y[32] = d * ((x[i].scales[1] & 0xF) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4)); - } else { - y[ 0] = d * ((x[i].scales[0] >> 4) - 8) * ((int8_t)((q >> 0) & 3) - ((h >> 0) & 1 ? 0 : 4)); - y[32] = d * ((x[i].scales[1] >> 4) - 8) * ((int8_t)((q >> 4) & 3) - ((h >> 4) & 1 ? 0 : 4)); - } -#endif - } -#if QK_K == 256 static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m) { if (j < 4) { d = q[j] & 63; m = q[j + 4] & 63; @@ -4289,7 +4252,6 @@ static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8 m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); } } -#endif template static void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restrict__ yy, @@ -4298,7 +4260,6 @@ static void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restri const int i = item_ct1.get_group(2); -#if QK_K == 256 // assume 32 threads const int tid = item_ct1.get_local_id(2); const int il = tid/8; @@ -4322,15 +4283,6 @@ static void dequantize_block_q4_K(const void * __restrict__ vx, dst_t * __restri y[l + 0] = d1 * (q[l] & 0xF) - m1; y[l +32] = d2 * (q[l] >> 4) - m2; } -#else - const int tid = item_ct1.get_local_id(2); - const uint8_t * q = x[i].qs; - dst_t * y = yy + i*QK_K; - const float d = (float)x[i].dm[0]; - const float m = (float)x[i].dm[1]; - y[tid+ 0] = d * (x[i].scales[0] & 0xF) * (q[tid] & 0xF) - m * (x[i].scales[0] >> 4); - y[tid+32] = d * (x[i].scales[1] & 0xF) * (q[tid] >> 4) - m * (x[i].scales[1] >> 4); -#endif } template @@ -4340,7 +4292,6 @@ static void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restri const int i = item_ct1.get_group(2); -#if QK_K == 256 // assume 64 threads - this is very slightly better than the one below const int tid = item_ct1.get_local_id(2); const int il = tid/16; // il is in 0...3 @@ -4367,18 +4318,6 @@ static void dequantize_block_q5_K(const void * __restrict__ vx, dst_t * __restri hm <<= 1; y[32] = d2 * ((ql[ 0] >> 4) + (qh[ 0] & hm ? 16 : 0)) - m2; y[33] = d2 * ((ql[ 1] >> 4) + (qh[ 1] & hm ? 16 : 0)) - m2; -#else - const int tid = item_ct1.get_local_id(2); - const uint8_t q = x[i].qs[tid]; - const int im = tid/8; // 0...3 - const int in = tid%8; // 0...7 - const int is = tid/16; // 0 or 1 - const uint8_t h = x[i].qh[in] >> im; - const float d = x[i].d; - dst_t * y = yy + i*QK_K + tid; - y[ 0] = d * x[i].scales[is+0] * ((q & 0xF) - ((h >> 0) & 1 ? 0 : 16)); - y[32] = d * x[i].scales[is+2] * ((q >> 4) - ((h >> 4) & 1 ? 0 : 16)); -#endif } template @@ -4387,7 +4326,6 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri const block_q6_K * x = (const block_q6_K *) vx; const int i = item_ct1.get_group(2); -#if QK_K == 256 // assume 64 threads - this is very slightly better than the one below const int tid = item_ct1.get_local_id(2); @@ -4407,24 +4345,6 @@ static void dequantize_block_q6_K(const void * __restrict__ vx, dst_t * __restri y[32] = d * sc[2] * ((int8_t)((ql[32] & 0xF) | (((qh >> 2) & 3) << 4)) - 32); y[64] = d * sc[4] * ((int8_t)((ql[ 0] >> 4) | (((qh >> 4) & 3) << 4)) - 32); y[96] = d * sc[6] * ((int8_t)((ql[32] >> 4) | (((qh >> 6) & 3) << 4)) - 32); -#else - - // assume 32 threads - const int tid = item_ct1.get_local_id(2); - const int ip = tid/16; // 0 or 1 - const int il = tid - 16*ip; // 0...15 - - dst_t * y = yy + i*QK_K + 16*ip + il; - - const float d = x[i].d; - - const uint8_t ql = x[i].ql[16*ip + il]; - const uint8_t qh = x[i].qh[il] >> (2*ip); - const int8_t * sc = x[i].scales; - - y[ 0] = d * sc[ip+0] * ((int8_t)((ql & 0xF) | (((qh >> 0) & 3) << 4)) - 32); - y[32] = d * sc[ip+2] * ((int8_t)((ql >> 4) | (((qh >> 4) & 3) << 4)) - 32); -#endif } template @@ -4438,7 +4358,6 @@ static void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __res const block_iq2_xxs * x = (const block_iq2_xxs *) vx; const int tid = item_ct1.get_local_id(2); -#if QK_K == 256 const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -4449,10 +4368,6 @@ static void dequantize_block_iq2_xxs(const void * __restrict__ vx, dst_t * __res const float d = (float)x[i].d * (0.5f + (aux32 >> 28)) * 0.25f; const uint8_t signs = ksigns_iq2xs_ptr[(aux32 >> 7*il) & 127]; for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs_ptr[j] ? -1.f : 1.f); -#else - assert(false); -#endif - } template @@ -4466,7 +4381,6 @@ static void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __rest const block_iq2_xs * x = (const block_iq2_xs *) vx; const int tid = item_ct1.get_local_id(2); -#if QK_K == 256 const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -4475,10 +4389,6 @@ static void dequantize_block_iq2_xs(const void * __restrict__ vx, dst_t * __rest const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; const uint8_t signs = ksigns_iq2xs[q2[il] >> 9]; for (int j = 0; j < 8; ++j) y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); -#else - assert(false); -#endif - } template @@ -4490,7 +4400,6 @@ dequantize_block_iq2_s(const void *__restrict__ vx, dst_t *__restrict__ yy, const block_iq2_s * x = (const block_iq2_s *) vx; const int tid = item_ct1.get_local_id(2); -#if QK_K == 256 const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -4498,13 +4407,9 @@ dequantize_block_iq2_s(const void *__restrict__ vx, dst_t *__restrict__ yy, const float d = (float)x[i].d * (0.5f + ((x[i].scales[ib] >> 4*(il/2)) & 0xf)) * 0.25f; const uint8_t signs = x[i].qs[QK_K/8+4*ib+il]; #pragma unroll - for (int j = 0; j < 8; ++j) + for (int j = 0; j < 8; ++j) { y[j] = d * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); -#else - assert(false); - -#endif - + } } template @@ -4518,7 +4423,6 @@ static void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __res const block_iq3_xxs * x = (const block_iq3_xxs *) vx; const int tid = item_ct1.get_local_id(2); -#if QK_K == 256 const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -4533,10 +4437,6 @@ static void dequantize_block_iq3_xxs(const void * __restrict__ vx, dst_t * __res y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); } -#else - assert(false); -#endif - } template @@ -4549,7 +4449,6 @@ dequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy, const block_iq3_s * x = (const block_iq3_s *) vx; const int tid = item_ct1.get_local_id(2); -#if QK_K == 256 const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -4563,10 +4462,6 @@ dequantize_block_iq3_s(const void *__restrict__ vx, dst_t *__restrict__ yy, y[j+0] = d * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); y[j+4] = d * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); } -#else - assert(false); -#endif - } template @@ -4579,7 +4474,6 @@ dequantize_block_iq1_s(const void *__restrict__ vx, dst_t *__restrict__ yy, const block_iq1_s * x = (const block_iq1_s *) vx; const int tid = item_ct1.get_local_id(2); -#if QK_K == 256 const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -4593,10 +4487,6 @@ dequantize_block_iq1_s(const void *__restrict__ vx, dst_t *__restrict__ yy, for (int j = 0; j < 8; ++j) { y[j] = d * (q[j] + delta); } -#else - assert(false); -#endif - } template @@ -4609,7 +4499,6 @@ dequantize_block_iq1_m(const void *__restrict__ vx, dst_t *__restrict__ yy, const block_iq1_m * x = (const block_iq1_m *) vx; const int tid = item_ct1.get_local_id(2); -#if QK_K == 256 const int il = tid/8; // 0...3 const int ib = tid%8; // 0...7 dst_t * y = yy + i*QK_K + 32*ib + 8*il; @@ -4627,10 +4516,6 @@ dequantize_block_iq1_m(const void *__restrict__ vx, dst_t *__restrict__ yy, for (int j = 0; j < 8; ++j) { y[j] = d * (q[j] + delta); } -#else - assert(false); -#endif - } template @@ -4704,7 +4589,6 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, float tmp = 0; // partial sum for thread in warp -#if QK_K == 256 const int tid = item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...15 const int ix = @@ -4755,42 +4639,6 @@ static void dequantize_mul_mat_vec_q2_k(const void *__restrict__ vx, tmp += dall * sum1 - dmin * sum2; } -#else - const int tid = item_ct1.get_local_id(2) / - (2 * K_QUANTS_PER_ITERATION); // 0...15 or 0...7 - const int ix = item_ct1.get_local_id(2) % - (2 * K_QUANTS_PER_ITERATION); // 0....1 or 0...3 - const int offset = tid * K_QUANTS_PER_ITERATION; - - uint32_t uaux[2]; - const uint8_t * d = (const uint8_t *)uaux; - - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - - const float * y = yy + i * QK_K + offset; - const uint8_t * q = x[i].qs + offset; - const uint32_t * s = (const uint32_t *)x[i].scales; - - uaux[0] = s[0] & 0x0f0f0f0f; - uaux[1] = (s[0] >> 4) & 0x0f0f0f0f; - - const sycl::float2 dall = - x[i].dm.convert(); - - float sum1 = 0, sum2 = 0; - for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) { - const uint8_t ql = q[l]; - sum1 += y[l+ 0] * d[0] * ((ql >> 0) & 3) - + y[l+16] * d[1] * ((ql >> 2) & 3) - + y[l+32] * d[2] * ((ql >> 4) & 3) - + y[l+48] * d[3] * ((ql >> 6) & 3); - sum2 += y[l+0] * d[4] + y[l+16] * d[5] + y[l+32] * d[6] + y[l+48] * d[7]; - } - tmp += dall.x() * sum1 - dall.y() * sum2; - } - -#endif // sum up partial sums and write back result #pragma unroll @@ -4828,8 +4676,6 @@ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx, float tmp = 0; // partial sum for thread in warp -#if QK_K == 256 - const uint16_t kmask1 = 0x0303; const uint16_t kmask2 = 0x0f0f; @@ -4882,34 +4728,6 @@ static void dequantize_mul_mat_vec_q3_k(const void *__restrict__ vx, tmp += d * sum; } -#else - - const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15 or 0...7 - const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0....1 or 0...3 - const int offset = tid * K_QUANTS_PER_ITERATION; // 0...15 or 0...14 - const int in = offset/8; // 0 or 1 - const int im = offset%8; // 0...7 - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - - const float * y = yy + i * QK_K + offset; - const uint8_t * q = x[i].qs + offset; - const uint8_t * s = x[i].scales; - - const float dall = (float)x[i].d; - - float sum = 0; - for (int l = 0; l < K_QUANTS_PER_ITERATION; ++l) { - const uint8_t hl = x[i].hmask[im+l] >> in; - const uint8_t ql = q[l]; - sum += y[l+ 0] * dall * ((s[0] & 0xF) - 8) * ((int8_t)((ql >> 0) & 3) - ((hl >> 0) & 1 ? 0 : 4)) - + y[l+16] * dall * ((s[0] >> 4) - 8) * ((int8_t)((ql >> 2) & 3) - ((hl >> 2) & 1 ? 0 : 4)) - + y[l+32] * dall * ((s[1] & 0xF) - 8) * ((int8_t)((ql >> 4) & 3) - ((hl >> 4) & 1 ? 0 : 4)) - + y[l+48] * dall * ((s[1] >> 4) - 8) * ((int8_t)((ql >> 6) & 3) - ((hl >> 6) & 1 ? 0 : 4)); - } - tmp += sum; - } -#endif // sum up partial sums and write back result #pragma unroll @@ -4944,7 +4762,6 @@ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, const block_q4_K * x = (const block_q4_K *)vx + ib0; -#if QK_K == 256 const uint16_t kmask1 = 0x3f3f; const uint16_t kmask2 = 0x0f0f; const uint16_t kmask3 = 0xc0c0; @@ -5033,36 +4850,6 @@ static void dequantize_mul_mat_vec_q4_k(const void *__restrict__ vx, #endif } -#else - const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15 - const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); - - const int step = tid * K_QUANTS_PER_ITERATION; - - uint16_t aux16[2]; - const uint8_t * s = (const uint8_t *)aux16; - - float tmp = 0; - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - const uint8_t * q = x[i].qs + step; - const float * y = yy + i*QK_K + step; - const uint16_t * a = (const uint16_t *)x[i].scales; - aux16[0] = a[0] & 0x0f0f; - aux16[1] = (a[0] >> 4) & 0x0f0f; - const float d = (float)x[i].dm[0]; - const float m = (float)x[i].dm[1]; - float sum = 0.f; - for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) { - sum += y[j+ 0] * (d * s[0] * (q[j+ 0] & 0xF) - m * s[2]) - + y[j+16] * (d * s[0] * (q[j+16] & 0xF) - m * s[2]) - + y[j+32] * (d * s[1] * (q[j+ 0] >> 4) - m * s[3]) - + y[j+48] * (d * s[1] * (q[j+16] >> 4) - m * s[3]); - } - tmp += sum; - } - -#endif // sum up partial sums and write back result #pragma unroll @@ -5097,7 +4884,6 @@ static void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx, float tmp = 0; // partial sum for thread in warp -#if QK_K == 256 const uint16_t kmask1 = 0x3f3f; const uint16_t kmask2 = 0x0f0f; const uint16_t kmask3 = 0xc0c0; @@ -5174,30 +4960,6 @@ static void dequantize_mul_mat_vec_q5_k(const void *__restrict__ vx, dmin * smin; } -#else - const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...15 - const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); - const int step = tid * K_QUANTS_PER_ITERATION; - const int im = step/8; - const int in = step%8; - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - const uint8_t * q = x[i].qs + step; - const int8_t * s = x[i].scales; - const float * y = yy + i*QK_K + step; - const float d = x[i].d; - float sum = 0.f; - for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) { - const uint8_t h = x[i].qh[in+j] >> im; - sum += y[j+ 0] * d * s[0] * ((q[j+ 0] & 0xF) - ((h >> 0) & 1 ? 0 : 16)) - + y[j+16] * d * s[1] * ((q[j+16] & 0xF) - ((h >> 2) & 1 ? 0 : 16)) - + y[j+32] * d * s[2] * ((q[j+ 0] >> 4) - ((h >> 4) & 1 ? 0 : 16)) - + y[j+48] * d * s[3] * ((q[j+16] >> 4) - ((h >> 6) & 1 ? 0 : 16)); - } - tmp += sum; - } -#endif - // sum up partial sums and write back result #pragma unroll for (int mask = 16; mask > 0; mask >>= 1) { @@ -5224,8 +4986,6 @@ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const floa const block_q6_K * x = (const block_q6_K *)vx + ib0; -#if QK_K == 256 - const int tid = item_ct1.get_local_id(2) / K_QUANTS_PER_ITERATION; // 0...31 or 0...16 const int ix = @@ -5282,37 +5042,6 @@ static void dequantize_mul_mat_vec_q6_k(const void * __restrict__ vx, const floa } -#else - - const int tid = item_ct1.get_local_id(2)/(2*K_QUANTS_PER_ITERATION); // 0...7 - const int ix = item_ct1.get_local_id(2)%(2*K_QUANTS_PER_ITERATION); // 0...3 - - const int step = tid * K_QUANTS_PER_ITERATION; - - float tmp = 0; // partial sum for thread in warp - - for (int i = ix; i < num_blocks_per_row; i += 2*K_QUANTS_PER_ITERATION) { - - const float * y = yy + i * QK_K + step; - const uint8_t * ql = x[i].ql + step; - const uint8_t * qh = x[i].qh + step; - const int8_t * s = x[i].scales; - - const float d = x[i+0].d; - - float sum = 0; - for (int j = 0; j < K_QUANTS_PER_ITERATION; ++j) { - sum += y[j+ 0] * s[0] * d * ((int8_t)((ql[j+ 0] & 0xF) | ((qh[j] & 0x03) << 4)) - 32) - + y[j+16] * s[1] * d * ((int8_t)((ql[j+16] & 0xF) | ((qh[j] & 0x0c) << 2)) - 32) - + y[j+32] * s[2] * d * ((int8_t)((ql[j+ 0] >> 4) | ((qh[j] & 0x30) >> 0)) - 32) - + y[j+48] * s[3] * d * ((int8_t)((ql[j+16] >> 4) | ((qh[j] & 0xc0) >> 2)) - 32); - } - tmp += sum; - - } - -#endif - // sum up partial sums and write back result #pragma unroll for (int mask = 16; mask > 0; mask >>= 1) { @@ -6857,7 +6586,6 @@ static __dpct_inline__ float vec_dot_q4_K_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { -#ifndef GGML_QKK_64 const block_q4_K * bq4_K = (const block_q4_K *) vbq; int v[2]; @@ -6899,52 +6627,6 @@ vec_dot_q4_K_q8_1(const void *__restrict__ vbq, } return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, bq4_K->dm, d8); - -#else - -#if __SYCL_ARCH__ >= VER_4VEC // lowest compute capability for integer intrinsics - const block_q4_K * bq4_K = (const block_q4_K *) vbq; - - float sumf_d = 0.0f; - float sumf_m = 0.0f; - - uint16_t aux16[2]; - const uint8_t * s = (const uint8_t *)aux16; - - const uint16_t * a = (const uint16_t *)bq4_K->scales; - aux16[0] = a[0] & 0x0f0f; - aux16[1] = (a[0] >> 4) & 0x0f0f; - - const float dall = bq4_K->dm[0]; - const float dmin = bq4_K->dm[1]; - - const float d8_1 = bq8_1[0].ds[0]; - const float d8_2 = bq8_1[1].ds[1]; - - const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2)); - const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4); - const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2)); - const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4); - - const int * q4 = (const int *)bq4_K->qs + (iqs/2); - const int v1 = q4[0]; - const int v2 = q4[4]; - - const int dot1 = dpct::dp4a(ui2, v2 & 0x0f0f0f0f, dpct::dp4a(ui1, v1 & 0x0f0f0f0f, 0)); - const int dot2 = dpct::dp4a(ui4, (v2 >> 4) & 0x0f0f0f0f, dpct::dp4a(ui3, (v1 >> 4) & 0x0f0f0f0f, 0)); - const int dot3 = dpct::dp4a(0x01010101, ui2, dpct::dp4a(0x01010101, ui1, 0)); - const int dot4 = dpct::dp4a(0x01010101, ui4, dpct::dp4a(0x01010101, ui3, 0)); - - sumf_d += d8_1 * (dot1 * s[0]) + d8_2 * (dot2 * s[1]); - sumf_m += d8_1 * (dot3 * s[2]) + d8_2 * (dot4 * s[3]); - - return dall * sumf_d - dmin * sumf_m; - -#else - bad_arch(); -#endif // __SYCL_ARCH__ >= VER_4VEC - -#endif } template @@ -7003,11 +6685,7 @@ load_tiles_q4_K(const void *__restrict__ vx, int *__restrict__ x_ql, const block_q4_K * bxi = bx0 + i*blocks_per_row + kbxd; -#if QK_K == 256 x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = bxi->dm; -#else - x_dm[i * (WARP_SIZE/QI4_K) + i / QI4_K + kbxd] = {bxi->dm[0], bxi->dm[1]}; -#endif } #pragma unroll @@ -7050,7 +6728,6 @@ static __dpct_inline__ float vec_dot_q5_K_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { -#ifndef GGML_QKK_64 const block_q5_K * bq5_K = (const block_q5_K *) vbq; int vl[2]; @@ -7092,48 +6769,6 @@ vec_dot_q5_K_q8_1(const void *__restrict__ vbq, } return vec_dot_q5_K_q8_1_impl_vmmq(vl, vh, u, sc, m, bq5_K->dm, d8); - -#else - -#if __SYCL_ARCH__ >= VER_4VEC // lowest compute capability for integer intrinsics - const block_q5_K * bq5_K = (const block_q5_K *) vbq; - - const int8_t * s = bq5_K->scales; - - const float d = bq5_K->d; - - const float d8_1 = bq8_1[0].ds[0]; - const float d8_2 = bq8_1[1].ds[1]; - - const int ui1 = *((const int *)bq8_1[0].qs + (iqs/2)); - const int ui2 = *((const int *)bq8_1[0].qs + (iqs/2) + 4); - const int ui3 = *((const int *)bq8_1[1].qs + (iqs/2)); - const int ui4 = *((const int *)bq8_1[1].qs + (iqs/2) + 4); - - const int * ql = (const int *)bq5_K->qs + (iqs/2); - const int vl1 = ql[0]; - const int vl2 = ql[4]; - - const int step = 4 * (iqs/2); // 0, 4, 8, 12 - const int im = step/8; // = 0 for iqs = 0, 2, = 1 for iqs = 4, 6 - const int in = step%8; // 0, 4, 0, 4 - const int vh = (*((const int *)(bq5_K->qh + in))) >> im; - - const int v1 = (((vh << 4) & 0x10101010) ^ 0x10101010) | ((vl1 >> 0) & 0x0f0f0f0f); - const int v2 = (((vh << 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 0) & 0x0f0f0f0f); - const int v3 = (((vh >> 0) & 0x10101010) ^ 0x10101010) | ((vl1 >> 4) & 0x0f0f0f0f); - const int v4 = (((vh >> 2) & 0x10101010) ^ 0x10101010) | ((vl2 >> 4) & 0x0f0f0f0f); - - const float sumf_d = d8_1 * (dpct::dp4a(ui1, v1, 0) * s[0] + dpct::dp4a(ui2, v2, 0) * s[1]) - + d8_2 * (dpct::dp4a(ui3, v3, 0) * s[2] + dpct::dp4a(ui4, v4, 0) * s[3]); - - return d * sumf_d; - -#else - bad_arch(); -#endif // __SYCL_ARCH__ >= VER_4VEC - -#endif } template @@ -7205,9 +6840,7 @@ load_tiles_q5_K(const void *__restrict__ vx, int *__restrict__ x_ql, const block_q5_K * bxi = bx0 + i*blocks_per_row + kbxd; -#if QK_K == 256 x_dm[i * (WARP_SIZE/QI5_K) + i / QI5_K + kbxd] = bxi->dm; -#endif } #pragma unroll @@ -7387,7 +7020,6 @@ vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, const uint64_t *iq2xxs_grid, const uint8_t *ksigns_iq2xs, const uint8_t *kmask_iq2xs) { -#if QK_K == 256 const block_iq2_xxs * bq2 = (const block_iq2_xxs *) vbq; #if QR2_XXS == 8 @@ -7428,10 +7060,6 @@ vec_dot_iq2_xxs_q8_1(const void *__restrict__ vbq, } return d * (sumi1 + sumi2); #endif -#else - assert(false); - return 0.f; -#endif } static __dpct_inline__ float @@ -7440,7 +7068,6 @@ vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq, const uint64_t *iq2xs_grid, const uint64_t *ksigns64) { #if DPCT_COMPATIBILITY_TEMP >= \ MIN_CC_DP4A // lowest compute capability for integer intrinsics -#if QK_K == 256 const block_iq2_xs * bq2 = (const block_iq2_xs *) vbq; const int ib32 = iqs; @@ -7478,16 +7105,11 @@ vec_dot_iq2_xs_q8_1(const void *__restrict__ vbq, assert(false); return 0.f; #endif -#else - assert(false); - return 0.f; -#endif } static __dpct_inline__ float vec_dot_iq2_s_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { -#if QK_K == 256 const block_iq2_s * bq2 = (const block_iq2_s *) vbq; const int ib32 = iqs; @@ -7531,9 +7153,6 @@ vec_dot_iq2_s_q8_1(const void *__restrict__ vbq, } const float d = (float)bq2->d * bq8_1[ib32].ds[0] * 0.25f; return d * ((0.5f + ls1) * sumi1 + (0.5f + ls2) * sumi2); -#else - assert(false); -#endif } static __dpct_inline__ float @@ -7542,7 +7161,6 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, const uint32_t *iq3xxs_grid, const uint64_t *ksigns64) { #if DPCT_COMPATIBILITY_TEMP >= \ MIN_CC_DP4A // lowest compute capability for integer intrinsics -#if QK_K == 256 const block_iq3_xxs * bq2 = (const block_iq3_xxs *) vbq; const int ib32 = iqs; @@ -7570,17 +7188,12 @@ vec_dot_iq3_xxs_q8_1(const void *__restrict__ vbq, assert(false); return 0.f; #endif -#else - assert(false); - return 0.f; -#endif } static __dpct_inline__ float vec_dot_iq3_s_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, const uint32_t *iq3s_grid) { -#if QK_K == 256 const block_iq3_s * bq2 = (const block_iq3_s *) vbq; const int ib32 = iqs; @@ -7609,16 +7222,12 @@ vec_dot_iq3_s_q8_1(const void *__restrict__ vbq, (1 + 2 * ((bq2->scales[ib32 / 2] >> 4 * (ib32 % 2)) & 0xf)) * bq8_1[ib32].ds[0]; return d * sumi; -#else - assert(false); -#endif } static __dpct_inline__ float vec_dot_iq1_s_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs, const uint32_t *iq1s_grid_gpu) { -#if QK_K == 256 const block_iq1_s * bq1 = (const block_iq1_s *) vbq; const int ib32 = iqs; @@ -7637,15 +7246,11 @@ vec_dot_iq1_s_q8_1(const void *__restrict__ vbq, const float d = d1q * bq8_1[ib32].ds[0]; const float m = d1q * bq8_1[ib32].ds[1]; return d * sumi + m * delta; -#else - assert(false); -#endif } static __dpct_inline__ float vec_dot_iq1_m_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { -#if QK_K == 256 const block_iq1_m * bq1 = (const block_iq1_m *) vbq; const int ib32 = iqs; @@ -7670,9 +7275,6 @@ vec_dot_iq1_m_q8_1(const void *__restrict__ vbq, scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); const float d = (float)scale.f16 * bq8_1[ib32].ds[0]; return d * ((sumi[0] + sumf[0]) * (2*((sc[ib32/2] >> 6*(ib32%2)) & 0x7) + 1) + (sumi[1] + sumf[1]) * (2*((sc[ib32/2] >> (6*(ib32%2)+3)) & 0x7) + 1)); -#else - assert(false); -#endif } static __dpct_inline__ void get_int_from_table_16(const uint32_t &q4, @@ -7720,7 +7322,6 @@ static __dpct_inline__ float vec_dot_iq4_xs_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { -#if QK_K == 256 const block_iq4_xs * bq4 = (const block_iq4_xs *) vbq; const uint8_t * values = (const uint8_t *)kvalues_iq4nl; @@ -7738,9 +7339,6 @@ vec_dot_iq4_xs_q8_1(const void *__restrict__ vbq, sumi2 = dpct::dp4a(v2, q8[j + 4], sumi2); } return d * (sumi1 + sumi2); -#else - assert(false); -#endif } template static void dequantize_row_q2_K_sycl(const void *vx, dst_t *y, const int k, dpct::queue_ptr stream) { const int nb = k / QK_K; -#if QK_K == 256 { dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); @@ -10215,27 +9812,12 @@ static void dequantize_row_q2_K_sycl(const void *vx, dst_t *y, const int k, dequantize_block_q2_K(vx, y, item_ct1); }); } -#else - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 32), - sycl::range<3>(1, 1, 32)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q2_K(vx, y, item_ct1); - }); - } - -#endif } template static void dequantize_row_q3_K_sycl(const void *vx, dst_t *y, const int k, dpct::queue_ptr stream) { const int nb = k / QK_K; -#if QK_K == 256 { dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); @@ -10247,19 +9829,6 @@ static void dequantize_row_q3_K_sycl(const void *vx, dst_t *y, const int k, dequantize_block_q3_K(vx, y, item_ct1); }); } -#else - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 32), - sycl::range<3>(1, 1, 32)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q3_K(vx, y, item_ct1); - }); - } -#endif } template @@ -10320,7 +9889,6 @@ template static void dequantize_row_q5_K_sycl(const void *vx, dst_t *y, const int k, dpct::queue_ptr stream) { const int nb = k / QK_K; -#if QK_K == 256 { dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); @@ -10332,27 +9900,12 @@ static void dequantize_row_q5_K_sycl(const void *vx, dst_t *y, const int k, dequantize_block_q5_K(vx, y, item_ct1); }); } -#else - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 32), - sycl::range<3>(1, 1, 32)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q5_K(vx, y, item_ct1); - }); - } - -#endif } template static void dequantize_row_q6_K_sycl(const void *vx, dst_t *y, const int k, dpct::queue_ptr stream) { const int nb = k / QK_K; -#if QK_K == 256 { dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); @@ -10364,20 +9917,6 @@ static void dequantize_row_q6_K_sycl(const void *vx, dst_t *y, const int k, dequantize_block_q6_K(vx, y, item_ct1); }); } -#else - { - dpct::has_capability_or_fail(stream->get_device(), - {sycl::aspect::fp16}); - - stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, nb) * - sycl::range<3>(1, 1, 32), - sycl::range<3>(1, 1, 32)), - [=](sycl::nd_item<3> item_ct1) { - dequantize_block_q6_K(vx, y, item_ct1); - }); - } - -#endif } template @@ -10529,9 +10068,6 @@ template static void dequantize_row_iq4_xs_sycl(const void *vx, dst_t *y, const int k, dpct::queue_ptr stream) { const int nb = (k + QK_K - 1) / QK_K; -#if QK_K == 64 - dequantize_row_iq4_nl_sycl(vx, y, k, stream); -#else { dpct::has_capability_or_fail(stream->get_device(), {sycl::aspect::fp16}); @@ -10546,7 +10082,6 @@ static void dequantize_row_iq4_xs_sycl(const void *vx, dst_t *y, const int k, }); }); } -#endif } @@ -12051,8 +11586,6 @@ static void ggml_mul_mat_q3_K_q8_1_sycl(const void *vx, const void *vy, const int nrows_y, const int nrows_dst, dpct::queue_ptr stream) try { -#if QK_K == 256 - int id; SYCL_CHECK( CHECK_TRY_ERROR(id = get_current_device_id())); @@ -12167,7 +11700,6 @@ static void ggml_mul_mat_q3_K_q8_1_sycl(const void *vx, const void *vy, }); } } -#endif } catch (sycl::exception const &exc) { std::cerr << exc.what() << "Exception caught at file:" << __FILE__ diff --git a/ggml.c b/ggml.c index d316e3d31..673c47748 100644 --- a/ggml.c +++ b/ggml.c @@ -871,22 +871,14 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { }, [GGML_TYPE_IQ4_XS] = { .type_name = "iq4_xs", -#if QK_K == 64 - .blck_size = QK4_NL, -#else .blck_size = QK_K, -#endif .type_size = sizeof(block_iq4_xs), .is_quantized = true, .to_float = (ggml_to_float_t) dequantize_row_iq4_xs, .from_float = quantize_row_iq4_xs, .from_float_reference = (ggml_from_float_t)quantize_row_iq4_xs_reference, .vec_dot = ggml_vec_dot_iq4_xs_q8_K, -#if QK_K == 64 - .vec_dot_type = GGML_TYPE_Q8_0, -#else .vec_dot_type = GGML_TYPE_Q8_K, -#endif .nrows = 1, }, [GGML_TYPE_Q8_K] = { @@ -22117,11 +22109,7 @@ size_t ggml_quantize_chunk( case GGML_TYPE_IQ1_S: result = quantize_iq1_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; -#if QK_K == 64 - case GGML_TYPE_IQ4_XS: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; -#else case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; -#endif case GGML_TYPE_F16: { size_t elemsize = sizeof(ggml_fp16_t); diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 42df2e4d0..67e23dcc1 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -905,9 +905,8 @@ class GGUFValueType(IntEnum): raise ValueError(f"Unknown type: {type(val)}") -# Note: Does not support GGML_QKK_64 -QK_K = 256 # Items here are (block size, type size) +QK_K = 256 GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = { GGMLQuantizationType.F32: (1, 4), GGMLQuantizationType.F16: (1, 2), diff --git a/llama.cpp b/llama.cpp index 34137c7ad..37b3d58c6 100644 --- a/llama.cpp +++ b/llama.cpp @@ -26,13 +26,9 @@ #ifdef GGML_USE_METAL # include "ggml-metal.h" #endif -#ifndef QK_K -# ifdef GGML_QKK_64 -# define QK_K 64 -# else -# define QK_K 256 -# endif -#endif + +// TODO: replace with ggml API call +#define QK_K 256 #ifdef __has_include #if __has_include() @@ -14308,8 +14304,6 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) && use_more_bits(qs.i_attention_wv, qs.n_attention_wv)) new_type = GGML_TYPE_Q6_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && qs.i_attention_wv < 4) new_type = GGML_TYPE_Q5_K; - else if (QK_K == 64 && (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S) && - (qs.i_attention_wv < qs.n_attention_wv/8 || qs.i_attention_wv >= 7*qs.n_attention_wv/8)) new_type = GGML_TYPE_Q6_K; if (qs.model.type == MODEL_70B) { // In the 70B model we have 8 heads sharing the same attn_v weights. As a result, the attn_v.weight tensor is // 8x smaller compared to attn_q.weight. Hence, we can get a nice boost in quantization accuracy with From d48c88cbd563b6cf0ce972e2f56796896e240736 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 23 May 2024 10:00:44 +0300 Subject: [PATCH 115/137] ggml : remove ggml_flash_attn and ggml_flash_ff (#7463) ggml-ci --- examples/finetune/finetune.cpp | 3 +- .../train-text-from-scratch.cpp | 3 +- ggml.c | 676 +----------------- ggml.h | 18 +- tests/test-grad0.cpp | 116 +-- 5 files changed, 47 insertions(+), 769 deletions(-) diff --git a/examples/finetune/finetune.cpp b/examples/finetune/finetune.cpp index 992426c1b..22425730f 100644 --- a/examples/finetune/finetune.cpp +++ b/examples/finetune/finetune.cpp @@ -643,7 +643,8 @@ static struct ggml_tensor * llama_build_lora_finetune_graphs( struct ggml_tensor * t15 = ggml_permute (ctx, t12, 0, 3, 1, 2); set_name(t15, "t15"); assert_shape_4d(t15, N, n_embd_head, n_head_kv, n_batch); struct ggml_tensor * t16; if (enable_flash_attn) { - t16 = ggml_flash_attn(ctx, t13, t14, t15, true); set_name(t16, "t16"); assert_shape_4d(t16, n_embd_head, N, n_head, n_batch); + GGML_ASSERT(false && "TODO: ggml_flash_attn_ext() not yet supported"); + //t16 = ggml_flash_attn(ctx, t13, t14, t15, true); set_name(t16, "t16"); assert_shape_4d(t16, n_embd_head, N, n_head, n_batch); } else { struct ggml_tensor * t16_0 = ggml_mul_mat (ctx, t14, t13); set_name(t16_0, "t16_0"); assert_shape_4d(t16_0, N, N, n_head, n_batch); struct ggml_tensor * t16_1 = ggml_scale_inplace (ctx, t16_0, kv_scale); set_name(t16_1, "t16_1"); assert_shape_4d(t16_1, N, N, n_head, n_batch); diff --git a/examples/train-text-from-scratch/train-text-from-scratch.cpp b/examples/train-text-from-scratch/train-text-from-scratch.cpp index 45bdfa8f5..e2f85c682 100644 --- a/examples/train-text-from-scratch/train-text-from-scratch.cpp +++ b/examples/train-text-from-scratch/train-text-from-scratch.cpp @@ -341,7 +341,8 @@ static struct ggml_tensor * llama_build_train_graphs( struct ggml_tensor * t15 = ggml_permute (ctx, t12, 0, 3, 1, 2); set_name(t15, "t15"); assert_shape_4d(t15, N, n_embd/n_head, n_head, n_batch); struct ggml_tensor * t16; if (enable_flash_attn) { - t16 = ggml_flash_attn(ctx, t13, t14, t15, true); set_name(t16, "t16"); assert_shape_4d(t16, n_embd/n_head, N, n_head, n_batch); + GGML_ASSERT(false && "TODO: ggml_flash_attn_ext() not yet supported"); + //t16 = ggml_flash_attn(ctx, t13, t14, t15, true); set_name(t16, "t16"); assert_shape_4d(t16, n_embd/n_head, N, n_head, n_batch); } else { struct ggml_tensor * t16_0 = ggml_mul_mat (ctx, t14, t13); set_name(t16_0, "t16_0"); assert_shape_4d(t16_0, N, N, n_head, n_batch); struct ggml_tensor * t16_1 = ggml_scale_inplace (ctx, t16_0, kv_scale); set_name(t16_1, "t16_1"); assert_shape_4d(t16_1, N, N, n_head, n_batch); diff --git a/ggml.c b/ggml.c index 673c47748..9e72b7a76 100644 --- a/ggml.c +++ b/ggml.c @@ -2670,9 +2670,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "ARGSORT", "LEAKY_RELU", - "FLASH_ATTN", "FLASH_ATTN_EXT", - "FLASH_FF", "FLASH_ATTN_BACK", "SSM_CONV", "SSM_SCAN", @@ -2698,7 +2696,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "CROSS_ENTROPY_LOSS_BACK", }; -static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76"); +static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -2760,9 +2758,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "argsort(x)", "leaky_relu(x)", - "flash_attn(x)", "flash_attn_ext(x)", - "flash_ff(x)", "flash_attn_back(x)", "ssm_conv(x)", "ssm_scan(x)", @@ -2788,7 +2784,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "cross_entropy_loss_back(x,y)", }; -static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 76"); +static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -6948,38 +6944,6 @@ struct ggml_tensor * ggml_top_k( return result; } -// ggml_flash_attn - -struct ggml_tensor * ggml_flash_attn( - struct ggml_context * ctx, - struct ggml_tensor * q, - struct ggml_tensor * k, - struct ggml_tensor * v, - bool masked) { - GGML_ASSERT(ggml_can_mul_mat(k, q)); - // TODO: check if vT can be multiplied by (k*qT) - - bool is_node = false; - - if (q->grad || k->grad || v->grad) { - is_node = true; - } - - //struct ggml_tensor * result = ggml_dup_tensor(ctx, q); - struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, q->ne); - - int32_t t = masked ? 1 : 0; - ggml_set_op_params(result, &t, sizeof(t)); - - result->op = GGML_OP_FLASH_ATTN; - result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; - result->src[0] = q; - result->src[1] = k; - result->src[2] = v; - - return result; -} - // ggml_flash_attn_ext struct ggml_tensor * ggml_flash_attn_ext( @@ -7039,38 +7003,6 @@ void ggml_flash_attn_ext_set_prec( ggml_set_op_params_i32(a, 2, prec_i32); // scale is on first pos, max_bias on second } -// ggml_flash_ff - -struct ggml_tensor * ggml_flash_ff( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b0, - struct ggml_tensor * b1, - struct ggml_tensor * c0, - struct ggml_tensor * c1) { - GGML_ASSERT(ggml_can_mul_mat(b0, a)); - // TODO: more checks - - bool is_node = false; - - if (a->grad || b0->grad || b1->grad || c0->grad || c1->grad) { - is_node = true; - } - - //struct ggml_tensor * result = ggml_dup_tensor(ctx, a); - struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, a->ne); - - result->op = GGML_OP_FLASH_FF; - result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL; - result->src[0] = a; - result->src[1] = b0; - result->src[2] = b1; - result->src[3] = c0; - result->src[4] = c1; - - return result; -} - // ggml_flash_attn_back struct ggml_tensor * ggml_flash_attn_back( @@ -7080,6 +7012,8 @@ struct ggml_tensor * ggml_flash_attn_back( struct ggml_tensor * v, struct ggml_tensor * d, bool masked) { + GGML_ASSERT(false && "TODO: adapt to ggml_flash_attn_ext() changes"); + GGML_ASSERT(ggml_can_mul_mat(k, q)); // TODO: check if vT can be multiplied by (k*qT) @@ -15709,400 +15643,6 @@ static void ggml_compute_forward_argsort( } } -// ggml_compute_forward_flash_attn - -static void ggml_compute_forward_flash_attn_f32( - const struct ggml_compute_params * params, - const bool masked, - struct ggml_tensor * dst) { - - const struct ggml_tensor * q = dst->src[0]; - const struct ggml_tensor * k = dst->src[1]; - const struct ggml_tensor * v = dst->src[2]; - - int64_t t0 = ggml_perf_time_us(); - UNUSED(t0); - - GGML_TENSOR_LOCALS(int64_t, neq, q, ne) - GGML_TENSOR_LOCALS(size_t, nbq, q, nb) - GGML_TENSOR_LOCALS(int64_t, nek, k, ne) - GGML_TENSOR_LOCALS(size_t, nbk, k, nb) - GGML_TENSOR_LOCALS(int64_t, nev, v, ne) - GGML_TENSOR_LOCALS(size_t, nbv, v, nb) - GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) - GGML_TENSOR_LOCALS(size_t, nb, dst, nb) - - const int ith = params->ith; - const int nth = params->nth; - - const int64_t D = neq0; - const int64_t N = neq1; - const int64_t P = nek1 - N; - const int64_t M = P + N; - - const int Mup = ggml_up(M, GGML_SOFT_MAX_UNROLL); - - GGML_ASSERT(ne0 == D); - GGML_ASSERT(ne1 == N); - GGML_ASSERT(P >= 0); - - GGML_ASSERT(nbq0 == sizeof(float)); - GGML_ASSERT(nbk0 == sizeof(float)); - GGML_ASSERT(nbv0 == sizeof(float)); - - GGML_ASSERT(neq0 == D); - GGML_ASSERT(nek0 == D); - GGML_ASSERT(nev1 == D); - - GGML_ASSERT(neq1 == N); - GGML_ASSERT(nek1 == N + P); - GGML_ASSERT(nev1 == D); - - // dst cannot be transposed or permuted - GGML_ASSERT(nb0 == sizeof(float)); - GGML_ASSERT(nb0 <= nb1); - GGML_ASSERT(nb1 <= nb2); - GGML_ASSERT(nb2 <= nb3); - - if (params->type == GGML_TASK_TYPE_INIT) { - return; - } - - if (params->type == GGML_TASK_TYPE_FINALIZE) { - return; - } - - // parallelize by q rows using ggml_vec_dot_f32 - - // total rows in q - const int nr = neq1*neq2*neq3; - - // rows per thread - const int dr = (nr + nth - 1)/nth; - - // row range for this thread - const int ir0 = dr*ith; - const int ir1 = MIN(ir0 + dr, nr); - - const float scale = 1.0f/sqrtf(D); - - //printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale); - - for (int ir = ir0; ir < ir1; ++ir) { - // q indices - const int iq3 = ir/(neq2*neq1); - const int iq2 = (ir - iq3*neq2*neq1)/neq1; - const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1); - - float * S = (float *) params->wdata + ith*(Mup + CACHE_LINE_SIZE_F32); - - for (int i = M; i < Mup; ++i) { - S[i] = -INFINITY; - } - - const int64_t masked_begin = masked ? (P + iq1 + 1) : M; - for (int64_t ic = 0; ic < masked_begin; ++ic) { - // k indices - const int ik3 = iq3; - const int ik2 = iq2 % nek2; - const int ik1 = ic; - - // S indices - const int i1 = ik1; - - ggml_vec_dot_f32(neq0, - S + i1, 0, - (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0, - (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1); - } - - // scale - ggml_vec_scale_f32(masked_begin, S, scale); - - for (int64_t i = masked_begin; i < M; i++) { - S[i] = -INFINITY; - } - - // softmax - // exclude known -INF S[..] values from max and loop - // dont forget to set their SW values to zero - { - float max = -INFINITY; - ggml_vec_max_f32(masked_begin, &max, S); - - ggml_float sum = 0.0; - { -#ifdef GGML_SOFT_MAX_ACCELERATE - max = -max; - vDSP_vsadd(S, 1, &max, S, 1, Mup); - vvexpf(S, S, &Mup); - ggml_vec_sum_f32(Mup, &sum, S); -#else - sum = ggml_vec_soft_max_f32(Mup, S, S, max); -#endif - } - - assert(sum > 0.0); - - sum = 1.0/sum; - ggml_vec_scale_f32(masked_begin, S, sum); - -#ifndef NDEBUG - for (int i = 0; i < masked_begin; ++i) { - assert(!isnan(S[i])); - assert(!isinf(S[i])); - } -#endif - } - - for (int64_t ic = 0; ic < nev1; ++ic) { - // dst indices - const int i1 = iq1; - const int i2 = iq2; - const int i3 = iq3; - - // v indices - const int iv2 = iq2 % nev2; - const int iv3 = iq3; - - ggml_vec_dot_f32(masked_begin, - (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0, - (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0, - S, 0, 1); - } - } -} - -static void ggml_compute_forward_flash_attn_f16( - const struct ggml_compute_params * params, - const bool masked, - struct ggml_tensor * dst) { - - const struct ggml_tensor * q = dst->src[0]; - const struct ggml_tensor * k = dst->src[1]; - const struct ggml_tensor * v = dst->src[2]; - - int64_t t0 = ggml_perf_time_us(); - UNUSED(t0); - - GGML_TENSOR_LOCALS(int64_t, neq, q, ne) - GGML_TENSOR_LOCALS(size_t, nbq, q, nb) - GGML_TENSOR_LOCALS(int64_t, nek, k, ne) - GGML_TENSOR_LOCALS(size_t, nbk, k, nb) - GGML_TENSOR_LOCALS(int64_t, nev, v, ne) - GGML_TENSOR_LOCALS(size_t, nbv, v, nb) - GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) - GGML_TENSOR_LOCALS(size_t, nb, dst, nb) - - const int ith = params->ith; - const int nth = params->nth; - - const int64_t D = neq0; - const int64_t N = neq1; - const int64_t P = nek1 - N; - const int64_t M = P + N; - - const int Mup = ggml_up(M, GGML_SOFT_MAX_UNROLL); - - GGML_ASSERT(ne0 == D); - GGML_ASSERT(ne1 == N); - GGML_ASSERT(P >= 0); - - GGML_ASSERT(nbq0 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t)); - - GGML_ASSERT(neq0 == D); - GGML_ASSERT(nek0 == D); - GGML_ASSERT(nev1 == D); - - GGML_ASSERT(neq1 == N); - GGML_ASSERT(nek1 == N + P); - GGML_ASSERT(nev1 == D); - - // dst cannot be transposed or permuted - GGML_ASSERT(nb0 == sizeof(float)); - GGML_ASSERT(nb0 <= nb1); - GGML_ASSERT(nb1 <= nb2); - GGML_ASSERT(nb2 <= nb3); - - if (params->type == GGML_TASK_TYPE_INIT) { - return; - } - - if (params->type == GGML_TASK_TYPE_FINALIZE) { - return; - } - - // parallelize by q rows using ggml_vec_dot_f32 - - // total rows in q - const int nr = neq1*neq2*neq3; - - // rows per thread - const int dr = (nr + nth - 1)/nth; - - // row range for this thread - const int ir0 = dr*ith; - const int ir1 = MIN(ir0 + dr, nr); - - const float scale = 1.0f/sqrtf(D); - - //printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale); - - for (int ir = ir0; ir < ir1; ++ir) { - // q indices - const int iq3 = ir/(neq2*neq1); - const int iq2 = (ir - iq3*neq2*neq1)/neq1; - const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1); - - float * S = (float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32); - - for (int i = M; i < Mup; ++i) { - S[i] = -INFINITY; - } - - if (GGML_VEC_DOT_UNROLL > 2 || nek1 % GGML_VEC_DOT_UNROLL != 0) { - for (int64_t ic = 0; ic < nek1; ++ic) { - // k indices - const int ik3 = iq3; - const int ik2 = iq2 % nek2; - const int ik1 = ic; - - // S indices - const int i1 = ik1; - - ggml_vec_dot_f16(neq0, - S + i1, 0, - (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0, - (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1); - } - } else { - for (int64_t ic = 0; ic < nek1; ic += GGML_VEC_DOT_UNROLL) { - // k indices - const int ik3 = iq3; - const int ik2 = iq2 % nek2; - const int ik1 = ic; - - // S indices - const int i1 = ik1; - - ggml_vec_dot_f16_unroll(neq0, nbk1, - S + i1, - ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), - (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3))); - } - } - - // scale - ggml_vec_scale_f32(nek1, S, scale); - - if (masked) { - for (int64_t i = P; i < M; i++) { - if (i > P + iq1) { - S[i] = -INFINITY; - } - } - } - - // softmax - // todo: exclude known -INF S[..] values from max and loop, assuming their results to be zero. - // dont forget to set their S values to zero - { - float max = -INFINITY; - ggml_vec_max_f32(M, &max, S); - - ggml_float sum = 0.0; - { -#ifdef GGML_SOFT_MAX_ACCELERATE - max = -max; - vDSP_vsadd(S, 1, &max, S, 1, Mup); - vvexpf(S, S, &Mup); - ggml_vec_sum_f32(Mup, &sum, S); -#else - sum = ggml_vec_soft_max_f32(Mup, S, S, max); -#endif - } - - assert(sum > 0.0); - - sum = 1.0/sum; - ggml_vec_scale_f32(M, S, sum); - -#ifndef NDEBUG - for (int i = 0; i < M; ++i) { - assert(!isnan(S[i])); - assert(!isinf(S[i])); - } -#endif - } - - ggml_fp16_t * S16 = (ggml_fp16_t *) ((float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32) + Mup); - - for (int64_t i = 0; i < M; i++) { - S16[i] = GGML_FP32_TO_FP16(S[i]); - } - - // todo: exclude known zero S[..] values from dot (reducing nev0 and increasing begin of v and S16). - if (GGML_VEC_DOT_UNROLL == 1 || (nev1 % GGML_VEC_DOT_UNROLL != 0)) { - for (int64_t ic = 0; ic < nev1; ++ic) { - // dst indices - const int i1 = iq1; - const int i2 = iq2; - const int i3 = iq3; - - // v indices - const int iv2 = iq2 % nev2; - const int iv3 = iq3; - - ggml_vec_dot_f16(nev0, - (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0, - (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0, - S16, 0, 1); - } - } else { - for (int64_t ic = 0; ic < nev1; ic += GGML_VEC_DOT_UNROLL) { - // dst indices - const int i1 = iq1; - const int i2 = iq2; - const int i3 = iq3; - - // v indices - const int iv2 = iq2 % nev2; - const int iv3 = iq3; - - ggml_vec_dot_f16_unroll(nev0, nbv1, - (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), - ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), - S16); - } - } - } -} - -static void ggml_compute_forward_flash_attn( - const struct ggml_compute_params * params, - const bool masked, - struct ggml_tensor * dst) { - - const struct ggml_tensor * q = dst->src[0]; - - switch (q->type) { - case GGML_TYPE_F16: - { - ggml_compute_forward_flash_attn_f16(params, masked, dst); - } break; - case GGML_TYPE_F32: - { - ggml_compute_forward_flash_attn_f32(params, masked, dst); - } break; - default: - { - GGML_ASSERT(false); - } break; - } -} - // ggml_compute_forward_flash_attn_ext static void ggml_compute_forward_flash_attn_ext_f16( @@ -16336,165 +15876,6 @@ static void ggml_compute_forward_flash_attn_ext( } } -// ggml_compute_forward_flash_ff - -static void ggml_compute_forward_flash_ff_f16( - const struct ggml_compute_params * params, - struct ggml_tensor * dst) { - - const struct ggml_tensor * a = dst->src[0]; // F16 - const struct ggml_tensor * b0 = dst->src[1]; // F16 fc_w - const struct ggml_tensor * b1 = dst->src[2]; // F32 fc_b - const struct ggml_tensor * c0 = dst->src[3]; // F16 proj_w - const struct ggml_tensor * c1 = dst->src[4]; // F32 proj_b - - int64_t t0 = ggml_perf_time_us(); - UNUSED(t0); - - GGML_TENSOR_LOCALS(int64_t, nea, a, ne) - GGML_TENSOR_LOCALS(size_t, nba, a, nb) - GGML_TENSOR_LOCALS(int64_t, neb0, b0, ne) - GGML_TENSOR_LOCALS(size_t, nbb0, b0, nb) - GGML_TENSOR_LOCALS(int64_t, neb1, b1, ne) - GGML_TENSOR_LOCALS(size_t, nbb1, b1, nb) - GGML_TENSOR_LOCALS(int64_t, nec0, c0, ne) - GGML_TENSOR_LOCALS(size_t, nbc0, c0, nb) - GGML_TENSOR_LOCALS(int64_t, nec1, c1, ne) - GGML_TENSOR_LOCALS(size_t, nbc1, c1, nb) - GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) - GGML_TENSOR_LOCALS(size_t, nb, dst, nb) - - const int ith = params->ith; - const int nth = params->nth; - - const int64_t D = nea0; - //const int64_t N = nea1; - const int64_t M = neb01; - - GGML_ASSERT(ne0 == nea0); - GGML_ASSERT(ne1 == nea1); - GGML_ASSERT(ne2 == nea2); - - GGML_ASSERT(nba0 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nbb00 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nbb10 == sizeof(float)); - GGML_ASSERT(nbc00 == sizeof(ggml_fp16_t)); - GGML_ASSERT(nbc10 == sizeof(float)); - - GGML_ASSERT(neb00 == D); - GGML_ASSERT(neb01 == M); - GGML_ASSERT(neb10 == M); - GGML_ASSERT(neb11 == 1); - - GGML_ASSERT(nec00 == M); - GGML_ASSERT(nec01 == D); - GGML_ASSERT(nec10 == D); - GGML_ASSERT(nec11 == 1); - - // dst cannot be transposed or permuted - GGML_ASSERT(nb0 == sizeof(float)); - GGML_ASSERT(nb0 <= nb1); - GGML_ASSERT(nb1 <= nb2); - GGML_ASSERT(nb2 <= nb3); - - if (params->type == GGML_TASK_TYPE_INIT) { - return; - } - - if (params->type == GGML_TASK_TYPE_FINALIZE) { - return; - } - - // parallelize by a rows using ggml_vec_dot_f32 - - // total rows in a - const int nr = nea1*nea2*nea3; - - // rows per thread - const int dr = (nr + nth - 1)/nth; - - // row range for this thread - const int ir0 = dr*ith; - const int ir1 = MIN(ir0 + dr, nr); - - for (int ir = ir0; ir < ir1; ++ir) { - // a indices - const int ia3 = ir/(nea2*nea1); - const int ia2 = (ir - ia3*nea2*nea1)/nea1; - const int ia1 = (ir - ia3*nea2*nea1 - ia2*nea1); - - float * S = (float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32); - - for (int64_t ic = 0; ic < neb01; ++ic) { - // b0 indices - const int ib03 = ia3; - const int ib02 = ia2; - const int ib01 = ic; - - // S indices - const int i1 = ib01; - - ggml_vec_dot_f16(nea0, - S + i1, 0, - (ggml_fp16_t *) ((char *) b0->data + (ib01*nbb01 + ib02*nbb02 + ib03*nbb03)), 0, - (ggml_fp16_t *) ((char *) a->data + ( ia1*nba1 + ia2*nba2 + ia3*nba3)), 0, 1); - } - - ggml_vec_add_f32(neb01, S, S, (float *) b1->data); - //ggml_vec_gelu_f32(neb01, S, S); - - ggml_fp16_t * S16 = (ggml_fp16_t *) ((float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32) + M); - - for (int64_t i = 0; i < M; i++) { - S16[i] = GGML_FP32_TO_FP16(S[i]); - } - - ggml_vec_gelu_f16(neb01, S16, S16); - - { - // dst indices - const int i1 = ia1; - const int i2 = ia2; - const int i3 = ia3; - - for (int64_t ic = 0; ic < nec01; ++ic) { - - ggml_vec_dot_f16(neb01, - (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0, - (ggml_fp16_t *) ((char *) c0->data + ( ic*nbc01 + i2*nbc02 + i3*nbc03)), 0, - S16, 0, 1); - } - - ggml_vec_add_f32(nec01, - (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)), - (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)), - (float *) c1->data); - } - } -} - -static void ggml_compute_forward_flash_ff( - const struct ggml_compute_params * params, - struct ggml_tensor * dst) { - - const struct ggml_tensor * b0 = dst->src[1]; - - switch (b0->type) { - case GGML_TYPE_F16: - { - ggml_compute_forward_flash_ff_f16(params, dst); - } break; - case GGML_TYPE_F32: - { - GGML_ASSERT(false); // TODO - } break; - default: - { - GGML_ASSERT(false); - } break; - } -} - // ggml_compute_forward_flash_attn_back static void ggml_compute_forward_flash_attn_back_f32( @@ -18065,21 +17446,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_leaky_relu(params, tensor); } break; - case GGML_OP_FLASH_ATTN: - { - const int32_t t = ggml_get_op_params_i32(tensor, 0); - GGML_ASSERT(t == 0 || t == 1); - const bool masked = t != 0; - ggml_compute_forward_flash_attn(params, masked, tensor); - } break; case GGML_OP_FLASH_ATTN_EXT: { ggml_compute_forward_flash_attn_ext(params, tensor->src[0], tensor->src[1], tensor->src[2], tensor->src[3], tensor); } break; - case GGML_OP_FLASH_FF: - { - ggml_compute_forward_flash_ff(params, tensor); - } break; case GGML_OP_FLASH_ATTN_BACK: { int32_t t = ggml_get_op_params_i32(tensor, 0); @@ -19086,7 +18456,6 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor { GGML_ASSERT(false); // TODO: not implemented } break; - case GGML_OP_FLASH_ATTN: case GGML_OP_FLASH_ATTN_EXT: { struct ggml_tensor * flash_grad = NULL; @@ -19140,10 +18509,6 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor zero_table); } } break; - case GGML_OP_FLASH_FF: - { - GGML_ASSERT(false); // not supported - } break; case GGML_OP_FLASH_ATTN_BACK: { GGML_ASSERT(false); // not supported @@ -19830,15 +19195,10 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads, int n_cur_ { n_tasks = n_threads; } break; - case GGML_OP_FLASH_ATTN: case GGML_OP_FLASH_ATTN_EXT: { n_tasks = n_threads; } break; - case GGML_OP_FLASH_FF: - { - n_tasks = n_threads; - } break; case GGML_OP_FLASH_ATTN_BACK: { n_tasks = n_threads; @@ -20235,40 +19595,12 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02*ne03; cur += sizeof(ggml_fp16_t)*ne10*ne11*ne12; } break; - case GGML_OP_FLASH_ATTN: - { - const int64_t ne11 = ggml_up(node->src[1]->ne[1], GGML_SOFT_MAX_UNROLL); - - if (node->src[1]->type == GGML_TYPE_F32) { - cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1) - cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2 - } else if (node->src[1]->type == GGML_TYPE_F16) { - cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1) - cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2 - } else if (node->src[1]->type == GGML_TYPE_BF16) { - cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1) - cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2 - } - } break; case GGML_OP_FLASH_ATTN_EXT: { const int64_t ne00 = node->src[0]->ne[0]; // D cur = 3*sizeof(float)*ne00*n_tasks; // 3x head size/thread } break; - case GGML_OP_FLASH_FF: - { - if (node->src[1]->type == GGML_TYPE_F32) { - cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1) - cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2 - } else if (node->src[1]->type == GGML_TYPE_F16) { - cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1) - cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2 - } else if (node->src[1]->type == GGML_TYPE_BF16) { - cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1) - cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2 - } - } break; case GGML_OP_FLASH_ATTN_BACK: { const int64_t D = node->src[0]->ne[0]; diff --git a/ggml.h b/ggml.h index 08835042c..be81e0c52 100644 --- a/ggml.h +++ b/ggml.h @@ -481,9 +481,7 @@ extern "C" { GGML_OP_ARGSORT, GGML_OP_LEAKY_RELU, - GGML_OP_FLASH_ATTN, GGML_OP_FLASH_ATTN_EXT, - GGML_OP_FLASH_FF, GGML_OP_FLASH_ATTN_BACK, GGML_OP_SSM_CONV, GGML_OP_SSM_SCAN, @@ -1761,13 +1759,6 @@ extern "C" { struct ggml_tensor * a, int k); - GGML_API struct ggml_tensor * ggml_flash_attn( - struct ggml_context * ctx, - struct ggml_tensor * q, - struct ggml_tensor * k, - struct ggml_tensor * v, - bool masked); - #define GGML_KQ_MASK_PAD 32 // q: [n_embd, n_batch, n_head, 1] @@ -1788,6 +1779,7 @@ extern "C" { struct ggml_tensor * a, enum ggml_prec prec); + // TODO: needs to be adapted to ggml_flash_attn_ext GGML_API struct ggml_tensor * ggml_flash_attn_back( struct ggml_context * ctx, struct ggml_tensor * q, @@ -1796,14 +1788,6 @@ extern "C" { struct ggml_tensor * d, bool masked); - GGML_API struct ggml_tensor * ggml_flash_ff( - struct ggml_context * ctx, - struct ggml_tensor * a, - struct ggml_tensor * b0, - struct ggml_tensor * b1, - struct ggml_tensor * c0, - struct ggml_tensor * c1); - GGML_API struct ggml_tensor * ggml_ssm_conv( struct ggml_context * ctx, struct ggml_tensor * s, diff --git a/tests/test-grad0.cpp b/tests/test-grad0.cpp index 8ff76c891..21ca43be3 100644 --- a/tests/test-grad0.cpp +++ b/tests/test-grad0.cpp @@ -1515,90 +1515,50 @@ int main(int argc, const char ** argv) { } // flash_attn f32 - { - srand(seed); - const int nargs = 3; + // TODO: adapt to ggml_flash_attn_ext() changes + //{ + // srand(seed); + // const int nargs = 3; - int64_t ne2[4]; + // int64_t ne2[4]; - get_random_dims(ne2, 4); - int64_t D = ne2[0]; - int64_t N = ne2[1]; - int64_t M = ne2[2] + N; - int64_t B = ne2[3]; + // get_random_dims(ne2, 4); + // int64_t D = ne2[0]; + // int64_t N = ne2[1]; + // int64_t M = ne2[2] + N; + // int64_t B = ne2[3]; - for (int masked = 0; masked <= 1; ++masked) { - for (int ndims = 2; ndims <= 4; ++ndims) { - int max_nrep = (ndims >= 3) ? 2 : 1; - for (int nrep = 1; nrep < max_nrep; ++nrep) { - int64_t neq[4] = { D, N, B*nrep, ne[3] }; - int64_t nek[4] = { D, M, B, ne[3] }; - int64_t nev[4] = { M, D, B, ne[3] }; - if (ndims == 2) { - neq[2] = 1; neq[3] = 1; - nek[2] = 1; nek[3] = 1; - nev[2] = 1; nev[3] = 1; - } else if (ndims == 3) { - neq[3] = 1; - nek[3] = 1; - nev[3] = 1; - } - x[0] = get_random_tensor_f32(ctx0, ndims, neq, -0.1250f, 0.1250f); - x[1] = get_random_tensor_f32(ctx0, ndims, nek, -0.1250f, 0.1250f); - x[2] = get_random_tensor_f32(ctx0, ndims, nev, -0.1250f, 0.1250f); - ggml_set_param(ctx0, x[0]); - ggml_set_param(ctx0, x[1]); - ggml_set_param(ctx0, x[2]); + // for (int masked = 0; masked <= 1; ++masked) { + // for (int ndims = 2; ndims <= 4; ++ndims) { + // int max_nrep = (ndims >= 3) ? 2 : 1; + // for (int nrep = 1; nrep < max_nrep; ++nrep) { + // int64_t neq[4] = { D, N, B*nrep, ne[3] }; + // int64_t nek[4] = { D, M, B, ne[3] }; + // int64_t nev[4] = { M, D, B, ne[3] }; + // if (ndims == 2) { + // neq[2] = 1; neq[3] = 1; + // nek[2] = 1; nek[3] = 1; + // nev[2] = 1; nev[3] = 1; + // } else if (ndims == 3) { + // neq[3] = 1; + // nek[3] = 1; + // nev[3] = 1; + // } + // x[0] = get_random_tensor_f32(ctx0, ndims, neq, -0.1250f, 0.1250f); + // x[1] = get_random_tensor_f32(ctx0, ndims, nek, -0.1250f, 0.1250f); + // x[2] = get_random_tensor_f32(ctx0, ndims, nev, -0.1250f, 0.1250f); + // ggml_set_param(ctx0, x[0]); + // ggml_set_param(ctx0, x[1]); + // ggml_set_param(ctx0, x[2]); - struct ggml_tensor * f = ggml_sum(ctx0, ggml_flash_attn(ctx0, x[0], x[1], x[2], (masked == 0))); + // struct ggml_tensor * f = ggml_sum(ctx0, ggml_flash_attn(ctx0, x[0], x[1], x[2], (masked == 0))); - check_gradient("flash_attn f32", ctx0, x, f, ndims, nargs, 1.5e-4f, 1e-3f, INFINITY); - } - } - } - } + // check_gradient("flash_attn f32", ctx0, x, f, ndims, nargs, 1.5e-4f, 1e-3f, INFINITY); + // } + // } + // } + //} - // flash_attn f16, not yet fully implemented - if(0) - { - srand(seed); - const int nargs = 3; - - int64_t ne2[4]; - - get_random_dims(ne2, 4); - int64_t D = ne2[0]; - int64_t N = ne2[1]; - int64_t M = ne2[2] + N; - int64_t B = ne2[3]; - - for (int masked = 0; masked <= 1; ++masked) { - for (int ndims = 2; ndims <= 4; ++ndims) { - int64_t neq[4] = { D, N, B, ne[3] }; - int64_t nek[4] = { D, M, B, ne[3] }; - int64_t nev[4] = { M, D, B, ne[3] }; - if (ndims == 2) { - neq[2] = 1; neq[3] = 1; - nek[2] = 1; nek[3] = 1; - nev[2] = 1; nev[3] = 1; - } else if (ndims == 3) { - neq[3] = 1; - nek[3] = 1; - nev[3] = 1; - } - x[0] = get_random_tensor_f16(ctx0, ndims, neq, -0.1250f, 0.1250f); - x[1] = get_random_tensor_f16(ctx0, ndims, nek, -0.1250f, 0.1250f); - x[2] = get_random_tensor_f16(ctx0, ndims, nev, -0.1250f, 0.1250f); - ggml_set_param(ctx0, x[0]); - ggml_set_param(ctx0, x[1]); - ggml_set_param(ctx0, x[2]); - - struct ggml_tensor * f = ggml_sum(ctx0, ggml_flash_attn(ctx0, x[0], x[1], x[2], (masked == 0))); - - check_gradient("flash_attn f16", ctx0, x, f, ndims, nargs, 1.5e-4f, 1e-3f, INFINITY); - } - } - } ggml_free(ctx0); } From 152da28ae54139e3754189b9e6e1c28e11277502 Mon Sep 17 00:00:00 2001 From: Brian Date: Thu, 23 May 2024 17:40:43 +1000 Subject: [PATCH 116/137] labeler.yml: add embedding label detector [no ci] (#7482) --- .github/labeler.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index fca60594f..a67f78044 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -62,6 +62,8 @@ server: ggml: - changed-files: - any-glob-to-any-file: + - ggml.c + - ggml.h - ggml-*.c - ggml-*.h - ggml-cuda/** @@ -71,3 +73,6 @@ nix: - "**/*.nix" - .github/workflows/nix-*.yml - .devops/nix/nixpkgs-instances.nix +embedding: + - changed-files: + - any-glob-to-any-file: examples/embedding/ From a61a94e543e3c6877c087e80fca27a0313ce5fd5 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 23 May 2024 12:38:18 +0300 Subject: [PATCH 117/137] llama : rename n_ctx -> cache.size, less confusing (#0) --- llama.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/llama.cpp b/llama.cpp index 37b3d58c6..3e09a2390 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2475,7 +2475,6 @@ static bool llama_kv_cache_init( static bool llama_kv_cache_find_slot( struct llama_kv_cache & cache, const struct llama_batch & batch) { - const uint32_t n_ctx = cache.size; const uint32_t n_tokens = batch.n_tokens; if (cache.recurrent) { @@ -2526,16 +2525,16 @@ static bool llama_kv_cache_find_slot( } // otherwise, one cell per token. - if (n_tokens > n_ctx) { - LLAMA_LOG_ERROR("%s: n_tokens=%d > n_ctx=%d\n", __func__, n_tokens, n_ctx); + if (n_tokens > cache.size) { + LLAMA_LOG_ERROR("%s: n_tokens=%d > cache.size=%d\n", __func__, n_tokens, cache.size); return false; } uint32_t n_tested = 0; while (true) { - if (cache.head + n_tokens > n_ctx) { - n_tested += n_ctx - cache.head; + if (cache.head + n_tokens > cache.size) { + n_tested += cache.size - cache.head; cache.head = 0; continue; } @@ -2554,7 +2553,7 @@ static bool llama_kv_cache_find_slot( break; } - if (n_tested >= n_ctx) { + if (n_tested >= cache.size) { //LLAMA_LOG_ERROR("%s: failed to find a slot for %d tokens\n", __func__, n_tokens); return false; } From 9b82476ee9e73065a759f8bcc4cf27ec7ab2ed8c Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Thu, 23 May 2024 11:49:53 +0200 Subject: [PATCH 118/137] Add missing inference support for GPTNeoXForCausalLM (Pythia and GPT-NeoX base models) (#7461) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * convert-hf : add conversion of bloom-style qkv tensor to gpt-style qkv (code borrowed from BloomModel) * llama : add inference support for LLM_ARCH_GPTNEOX * llama : add model types for every Pythia variant and GPT-NeoX Co-authored-by: Stanisław Szymczyk --- convert-hf-to-gguf.py | 38 +++++++ llama.cpp | 236 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 273 insertions(+), 1 deletion(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index daad1c4fc..5a00a5e89 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -673,6 +673,44 @@ class GPTNeoXModel(Model): self.gguf_writer.add_parallel_residual(self.hparams.get("use_parallel_residual", True)) self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_eps"]) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + del bid # unused + + n_head = self.hparams.get("n_head", self.hparams.get("num_attention_heads")) + n_embed = self.hparams.get("hidden_size", self.hparams.get("n_embed")) + + tensors: list[tuple[str, Tensor]] = [] + + if re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.weight", name): + # Map bloom-style qkv_linear to gpt-style qkv_linear + # bloom: https://github.com/huggingface/transformers/blob/main/src/transformers/models/bloom/modeling_bloom.py#L238-L252 # noqa + # gpt-2: https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt2/modeling_gpt2.py#L312 # noqa + qkv_weights = data_torch.reshape((n_head, 3, n_embed // n_head, n_embed)) + data_torch = torch.cat( + ( + qkv_weights[:, 0, :, :].reshape((-1, n_embed)), + qkv_weights[:, 1, :, :].reshape((-1, n_embed)), + qkv_weights[:, 2, :, :].reshape((-1, n_embed)), + ), + dim=0, + ) + logger.info("re-format attention.linear_qkv.weight") + elif re.match(r"gpt_neox\.layers\.\d+\.attention\.query_key_value\.bias", name): + qkv_bias = data_torch.reshape((n_head, 3, n_embed // n_head)) + data_torch = torch.cat( + ( + qkv_bias[:, 0, :].reshape((n_embed,)), + qkv_bias[:, 1, :].reshape((n_embed,)), + qkv_bias[:, 2, :].reshape((n_embed,)), + ), + dim=0, + ) + logger.info("re-format attention.linear_qkv.bias") + + tensors.append((self.map_tensor_name(name), data_torch)) + + return tensors + @Model.register("BloomForCausalLM") class BloomModel(Model): diff --git a/llama.cpp b/llama.cpp index 3e09a2390..5ff186a57 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1692,17 +1692,24 @@ static llama_state g_state; // available llama models enum e_model { MODEL_UNKNOWN, + MODEL_14M, MODEL_17M, MODEL_22M, MODEL_33M, + MODEL_70M, MODEL_109M, MODEL_137M, + MODEL_160M, MODEL_335M, + MODEL_410M, MODEL_0_5B, MODEL_1B, + MODEL_1_4B, MODEL_2B, + MODEL_2_8B, MODEL_3B, MODEL_4B, + MODEL_6_9B, MODEL_7B, MODEL_8B, MODEL_12B, @@ -1734,6 +1741,7 @@ static const size_t GiB = 1024*MiB; struct llama_hparams { bool vocab_only; bool rope_finetuned; + bool use_par_res; uint32_t n_vocab; uint32_t n_ctx_train; // context size the model was trained on @@ -3773,17 +3781,24 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { static const char * llama_model_type_name(e_model type) { switch (type) { + case MODEL_14M: return "14M"; case MODEL_17M: return "17M"; case MODEL_22M: return "22M"; case MODEL_33M: return "33M"; + case MODEL_70M: return "70M"; case MODEL_109M: return "109M"; case MODEL_137M: return "137M"; + case MODEL_160M: return "160M"; case MODEL_335M: return "335M"; + case MODEL_410M: return "410M"; case MODEL_0_5B: return "0.5B"; case MODEL_1B: return "1B"; + case MODEL_1_4B: return "1.4B"; case MODEL_2B: return "2B"; + case MODEL_2_8B: return "2.8B"; case MODEL_3B: return "3B"; case MODEL_4B: return "4B"; + case MODEL_6_9B: return "6.9B"; case MODEL_7B: return "7B"; case MODEL_8B: return "8B"; case MODEL_12B: return "12B"; @@ -4282,6 +4297,52 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_GPTNEOX: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + ml.get_key(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res); + switch (hparams.n_layer) { + case 6: + switch (hparams.n_ff) { + case 512: model.type = e_model::MODEL_14M; break; + case 2048: model.type = e_model::MODEL_70M; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + case 12: + switch (hparams.n_ff) { + case 3072: model.type = e_model::MODEL_160M; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + case 16: + switch (hparams.n_ff) { + case 8192: model.type = e_model::MODEL_1B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + case 24: + switch (hparams.n_ff) { + case 4096: model.type = e_model::MODEL_410M; break; + case 8192: model.type = e_model::MODEL_1_4B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + case 32: + switch (hparams.n_ff) { + case 10240: model.type = e_model::MODEL_2_8B; break; + case 16384: model.type = e_model::MODEL_6_9B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + case 36: + switch (hparams.n_ff) { + case 20480: model.type = e_model::MODEL_12B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + case 44: + switch (hparams.n_ff) { + case 24576: model.type = e_model::MODEL_20B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } break; default: (void)0; } @@ -6033,6 +6094,41 @@ static bool llm_load_tensors( layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); } } break; + case LLM_ARCH_GPTNEOX: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + // output + { + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}); + } + + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}); + + layer.wqkv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}); + layer.bqkv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}); + + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}); + + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}); + + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}); + layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}); + + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}); + layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); + } + } break; default: throw std::runtime_error("unknown architecture"); } @@ -10560,6 +10656,140 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_gptneox() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + const int64_t n_embd_head = hparams.n_embd_head_v; + const int64_t n_embd_gqa = hparams.n_embd_v_gqa(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = build_inp_pos(); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); + + for (int il = 0; il < n_layer; ++il) { + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, + model.layers[il].attn_norm_b, + LLM_NORM, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur); + cb(cur, "wqkv", il); + + cur = ggml_add(ctx0, cur, model.layers[il].bqkv); + cb(cur, "bqkv", il); + + struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd))); + struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd))); + struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa))); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf, + model.layers[il].wo, model.layers[il].bo, + Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + } + + // ffn + if (hparams.use_par_res) { + // attention and ffn are computed in parallel + // x = x + attn(ln1(x)) + ffn(ln2(x)) + + struct ggml_tensor * attn_out = cur; + + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, + NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, inpL); + cb(cur, "ffn_out", il); + + inpL = ggml_add(ctx0, cur, attn_out); + cb(inpL, "l_out", il); + } else { + // attention and ffn are computed sequentially + // x = x + attn(ln1(x)) + // x = x + ffn(ln2(x)) + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL); + cb(ffn_inp, "ffn_inp", il); + + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, + model.layers[il].ffn_norm_b, + LLM_NORM, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, + NULL, NULL, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, + NULL, + LLM_FFN_GELU, LLM_FFN_SEQ, cb, il); + cb(cur, "ffn_out", il); + + inpL = ggml_add(ctx0, cur, ffn_inp); + cb(inpL, "l_out", il); + } + } + + cur = llm_build_norm(ctx0, inpL, hparams, + model.output_norm, + model.output_norm_b, + LLM_NORM, cb, -1); + cb(cur, "result_norm", -1); + + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } }; static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector & ids) { @@ -10770,6 +11000,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_olmo(); } break; + case LLM_ARCH_GPTNEOX: + { + result = llm.build_gptneox(); + } break; default: GGML_ASSERT(false); } @@ -15762,7 +15996,6 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { // these models do not use RoPE case LLM_ARCH_GPT2: case LLM_ARCH_GPTJ: - case LLM_ARCH_GPTNEOX: case LLM_ARCH_MPT: case LLM_ARCH_REFACT: case LLM_ARCH_BLOOM: @@ -15798,6 +16031,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_PHI3: case LLM_ARCH_GEMMA: case LLM_ARCH_STARCODER2: + case LLM_ARCH_GPTNEOX: return LLAMA_ROPE_TYPE_NEOX; // all model arches should be listed explicitly here From dacfcebd6022175848e978f82811a244f1033038 Mon Sep 17 00:00:00 2001 From: Victor Nogueira Date: Thu, 23 May 2024 15:12:43 +0300 Subject: [PATCH 119/137] readme : add GPT-NeoX + Pythia to the list of supported models (#7491) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f4088c05e..ccd12e211 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,7 @@ Typically finetunes of the base models below are supported as well. - [x] [SEA-LION](https://huggingface.co/models?search=sea-lion) - [x] [GritLM-7B](https://huggingface.co/GritLM/GritLM-7B) + [GritLM-8x7B](https://huggingface.co/GritLM/GritLM-8x7B) - [x] [OLMo](https://allenai.org/olmo) +- [x] [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) + [Pythia](https://github.com/EleutherAI/pythia) (instructions for supporting more models: [HOWTO-add-model.md](./docs/HOWTO-add-model.md)) From 55ac3b7aeaf52f19786ed96e885d89521fc0f6c8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 23 May 2024 15:28:14 +0300 Subject: [PATCH 120/137] ci : use Pythia models instead of OpenLlama (#7470) * ci : start using Pythia models over OpenLlama ggml-ci * ci : disable q2_k ppl tests * ci : use convert-hf-to-gguf.py * ci : update gg_get_model * ci : fix convert outfile name ggml-ci * llama : gptneox arch use F32 attn prec ggml-ci --- ci/run.sh | 424 +++++++++++++++++++++++++++++++++++------------------- llama.cpp | 4 +- 2 files changed, 282 insertions(+), 146 deletions(-) diff --git a/ci/run.sh b/ci/run.sh index 79dcd0772..940299025 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -202,12 +202,15 @@ function gg_sum_test_scripts_release { } function gg_get_model { - local gguf_3b="$MNT/models/open-llama/3B-v2/ggml-model-f16.gguf" - local gguf_7b="$MNT/models/open-llama/7B-v2/ggml-model-f16.gguf" - if [[ -s $gguf_3b ]]; then - echo -n "$gguf_3b" - elif [[ -s $gguf_7b ]]; then - echo -n "$gguf_7b" + local gguf_0="$MNT/models/pythia/1.4B/ggml-model-f16.gguf" + local gguf_1="$MNT/models/pythia/2.8B/ggml-model-f16.gguf" + local gguf_2="$MNT/models/open-llama/7B-v2/ggml-model-f16.gguf" + if [[ -s $gguf_0 ]]; then + echo -n "$gguf_0" + elif [[ -s $gguf_1 ]]; then + echo -n "$gguf_1" + elif [[ -s $gguf_2 ]]; then + echo -n "$gguf_2" else echo >&2 "No model found. Can't run gg_run_ctest_with_model." exit 1 @@ -256,139 +259,6 @@ function gg_sum_ctest_with_model_release { gg_printf '```\n' } -# open_llama_3b_v2 - -function gg_run_open_llama_3b_v2 { - cd ${SRC} - - gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/config.json - gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/tokenizer.model - gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/tokenizer_config.json - gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/special_tokens_map.json - gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/resolve/main/pytorch_model.bin - gg_wget models-mnt/open-llama/3B-v2/ https://huggingface.co/openlm-research/open_llama_3b_v2/raw/main/generation_config.json - - gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip - unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/ - head -n 60 models-mnt/wikitext/wikitext-2-raw/wiki.test.raw > models-mnt/wikitext/wikitext-2-raw/wiki.test-60.raw - - path_models="../models-mnt/open-llama/3B-v2" - path_wiki="../models-mnt/wikitext/wikitext-2-raw" - - rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release - - set -e - - (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} -DLLAMA_QKK_64=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log - (time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log - - python3 ../convert.py ${path_models} - - model_f16="${path_models}/ggml-model-f16.gguf" - model_q8_0="${path_models}/ggml-model-q8_0.gguf" - model_q4_0="${path_models}/ggml-model-q4_0.gguf" - model_q4_1="${path_models}/ggml-model-q4_1.gguf" - model_q5_0="${path_models}/ggml-model-q5_0.gguf" - model_q5_1="${path_models}/ggml-model-q5_1.gguf" - model_q2_k="${path_models}/ggml-model-q2_k.gguf" - model_q3_k="${path_models}/ggml-model-q3_k.gguf" - model_q4_k="${path_models}/ggml-model-q4_k.gguf" - model_q5_k="${path_models}/ggml-model-q5_k.gguf" - model_q6_k="${path_models}/ggml-model-q6_k.gguf" - - wiki_test_60="${path_wiki}/wiki.test-60.raw" - - ./bin/quantize ${model_f16} ${model_q8_0} q8_0 - ./bin/quantize ${model_f16} ${model_q4_0} q4_0 - ./bin/quantize ${model_f16} ${model_q4_1} q4_1 - ./bin/quantize ${model_f16} ${model_q5_0} q5_0 - ./bin/quantize ${model_f16} ${model_q5_1} q5_1 - ./bin/quantize ${model_f16} ${model_q2_k} q2_k - ./bin/quantize ${model_f16} ${model_q3_k} q3_k - ./bin/quantize ${model_f16} ${model_q4_k} q4_k - ./bin/quantize ${model_f16} ${model_q5_k} q5_k - ./bin/quantize ${model_f16} ${model_q6_k} q6_k - - (time ./bin/main --model ${model_f16} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log - (time ./bin/main --model ${model_q8_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log - (time ./bin/main --model ${model_q4_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log - (time ./bin/main --model ${model_q4_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log - (time ./bin/main --model ${model_q5_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log - (time ./bin/main --model ${model_q5_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log - (time ./bin/main --model ${model_q2_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log - (time ./bin/main --model ${model_q3_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log - (time ./bin/main --model ${model_q4_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log - (time ./bin/main --model ${model_q5_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log - (time ./bin/main --model ${model_q6_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log - - (time ./bin/perplexity --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log - (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log - (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log - (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log - (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log - (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log - (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log - (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log - (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log - (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log - (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log - - (time ./bin/imatrix --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log - - (time ./bin/save-load-state --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log - (time ./bin/save-load-state -fa --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log - - function check_ppl { - qnt="$1" - ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1) - - if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then - printf ' - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl" - return 20 - fi - - printf ' - %s @ %s OK\n' "$qnt" "$ppl" - return 0 - } - - check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log - - cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log - - set +e -} - -function gg_sum_open_llama_3b_v2 { - gg_printf '### %s\n\n' "${ci}" - - gg_printf 'OpenLLaMA 3B-v2:\n' - gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" - gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)" - gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)" - gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)" - gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)" - gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)" - gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)" - gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)" - gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)" - gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)" - gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)" - gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)" - gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)" - gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)" - gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)" -} - # open_llama_7b_v2 # requires: GG_BUILD_CUDA @@ -417,7 +287,7 @@ function gg_run_open_llama_7b_v2 { (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} -DLLAMA_CUDA=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log (time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log - python3 ../convert.py ${path_models} + python3 ../convert.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf model_f16="${path_models}/ggml-model-f16.gguf" model_q8_0="${path_models}/ggml-model-q8_0.gguf" @@ -526,6 +396,272 @@ function gg_sum_open_llama_7b_v2 { gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)" } +# pythia_1.4b + +function gg_run_pythia_1_4b { + cd ${SRC} + + gg_wget models-mnt/pythia/1.4B/ https://huggingface.co/EleutherAI/pythia-1.4b/raw/main/config.json + gg_wget models-mnt/pythia/1.4B/ https://huggingface.co/EleutherAI/pythia-1.4b/raw/main/tokenizer.json + gg_wget models-mnt/pythia/1.4B/ https://huggingface.co/EleutherAI/pythia-1.4b/raw/main/tokenizer_config.json + gg_wget models-mnt/pythia/1.4B/ https://huggingface.co/EleutherAI/pythia-1.4b/raw/main/special_tokens_map.json + gg_wget models-mnt/pythia/1.4B/ https://huggingface.co/EleutherAI/pythia-1.4b/resolve/main/pytorch_model.bin + + gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip + unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/ + head -n 60 models-mnt/wikitext/wikitext-2-raw/wiki.test.raw > models-mnt/wikitext/wikitext-2-raw/wiki.test-60.raw + + path_models="../models-mnt/pythia/1.4B" + path_wiki="../models-mnt/wikitext/wikitext-2-raw" + + rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release + + set -e + + (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log + (time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log + + python3 ../convert-hf-to-gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf + + model_f16="${path_models}/ggml-model-f16.gguf" + model_q8_0="${path_models}/ggml-model-q8_0.gguf" + model_q4_0="${path_models}/ggml-model-q4_0.gguf" + model_q4_1="${path_models}/ggml-model-q4_1.gguf" + model_q5_0="${path_models}/ggml-model-q5_0.gguf" + model_q5_1="${path_models}/ggml-model-q5_1.gguf" + model_q2_k="${path_models}/ggml-model-q2_k.gguf" + model_q3_k="${path_models}/ggml-model-q3_k.gguf" + model_q4_k="${path_models}/ggml-model-q4_k.gguf" + model_q5_k="${path_models}/ggml-model-q5_k.gguf" + model_q6_k="${path_models}/ggml-model-q6_k.gguf" + + wiki_test_60="${path_wiki}/wiki.test-60.raw" + + ./bin/quantize ${model_f16} ${model_q8_0} q8_0 + ./bin/quantize ${model_f16} ${model_q4_0} q4_0 + ./bin/quantize ${model_f16} ${model_q4_1} q4_1 + ./bin/quantize ${model_f16} ${model_q5_0} q5_0 + ./bin/quantize ${model_f16} ${model_q5_1} q5_1 + ./bin/quantize ${model_f16} ${model_q2_k} q2_k + ./bin/quantize ${model_f16} ${model_q3_k} q3_k + ./bin/quantize ${model_f16} ${model_q4_k} q4_k + ./bin/quantize ${model_f16} ${model_q5_k} q5_k + ./bin/quantize ${model_f16} ${model_q6_k} q6_k + + (time ./bin/main --model ${model_f16} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log + (time ./bin/main --model ${model_q8_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log + (time ./bin/main --model ${model_q4_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log + (time ./bin/main --model ${model_q4_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log + (time ./bin/main --model ${model_q5_0} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log + (time ./bin/main --model ${model_q5_1} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log + (time ./bin/main --model ${model_q2_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log + (time ./bin/main --model ${model_q3_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log + (time ./bin/main --model ${model_q4_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log + (time ./bin/main --model ${model_q5_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log + (time ./bin/main --model ${model_q6_k} -s 1234 -n 64 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log + + (time ./bin/perplexity --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log + (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log + (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log + (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log + (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log + (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log + (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log + (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log + (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log + (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log + (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log + + (time ./bin/imatrix --model ${model_f16} -f ${wiki_test_60} -c 128 -b 128 --chunks 1 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log + + (time ./bin/save-load-state --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/save-load-state -fa --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + + function check_ppl { + qnt="$1" + ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1) + + if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then + printf ' - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl" + return 20 + fi + + printf ' - %s @ %s OK\n' "$qnt" "$ppl" + return 0 + } + + check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + #check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log # note: ppl > 20.0 for this quant and model + check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + + cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log + + set +e +} + +function gg_sum_pythia_1_4b { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Pythia 1.4B:\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)" + gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)" + gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)" + gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)" + gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)" + gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)" + gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)" + gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)" + gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)" + gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)" + gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)" + gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)" + gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)" + gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)" +} + +# pythia_2_8b +# requires: GG_BUILD_CUDA + +function gg_run_pythia_2_8b { + cd ${SRC} + + gg_wget models-mnt/pythia/2.8B/ https://huggingface.co/EleutherAI/pythia-2.8b/raw/main/config.json + gg_wget models-mnt/pythia/2.8B/ https://huggingface.co/EleutherAI/pythia-2.8b/raw/main/tokenizer.json + gg_wget models-mnt/pythia/2.8B/ https://huggingface.co/EleutherAI/pythia-2.8b/raw/main/tokenizer_config.json + gg_wget models-mnt/pythia/2.8B/ https://huggingface.co/EleutherAI/pythia-2.8b/raw/main/special_tokens_map.json + gg_wget models-mnt/pythia/2.8B/ https://huggingface.co/EleutherAI/pythia-2.8b/resolve/main/pytorch_model.bin + + gg_wget models-mnt/wikitext/ https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip + unzip -o models-mnt/wikitext/wikitext-2-raw-v1.zip -d models-mnt/wikitext/ + + path_models="../models-mnt/pythia/2.8B" + path_wiki="../models-mnt/wikitext/wikitext-2-raw" + + rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release + + set -e + + (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} -DLLAMA_CUDA=1 .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log + (time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log + + python3 ../convert-hf-to-gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf + + model_f16="${path_models}/ggml-model-f16.gguf" + model_q8_0="${path_models}/ggml-model-q8_0.gguf" + model_q4_0="${path_models}/ggml-model-q4_0.gguf" + model_q4_1="${path_models}/ggml-model-q4_1.gguf" + model_q5_0="${path_models}/ggml-model-q5_0.gguf" + model_q5_1="${path_models}/ggml-model-q5_1.gguf" + model_q2_k="${path_models}/ggml-model-q2_k.gguf" + model_q3_k="${path_models}/ggml-model-q3_k.gguf" + model_q4_k="${path_models}/ggml-model-q4_k.gguf" + model_q5_k="${path_models}/ggml-model-q5_k.gguf" + model_q6_k="${path_models}/ggml-model-q6_k.gguf" + + wiki_test="${path_wiki}/wiki.test.raw" + + ./bin/quantize ${model_f16} ${model_q8_0} q8_0 + ./bin/quantize ${model_f16} ${model_q4_0} q4_0 + ./bin/quantize ${model_f16} ${model_q4_1} q4_1 + ./bin/quantize ${model_f16} ${model_q5_0} q5_0 + ./bin/quantize ${model_f16} ${model_q5_1} q5_1 + ./bin/quantize ${model_f16} ${model_q2_k} q2_k + ./bin/quantize ${model_f16} ${model_q3_k} q3_k + ./bin/quantize ${model_f16} ${model_q4_k} q4_k + ./bin/quantize ${model_f16} ${model_q5_k} q5_k + ./bin/quantize ${model_f16} ${model_q6_k} q6_k + + (time ./bin/main --model ${model_f16} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log + (time ./bin/main --model ${model_q8_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log + (time ./bin/main --model ${model_q4_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log + (time ./bin/main --model ${model_q4_1} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log + (time ./bin/main --model ${model_q5_0} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log + (time ./bin/main --model ${model_q5_1} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log + (time ./bin/main --model ${model_q2_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log + (time ./bin/main --model ${model_q3_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log + (time ./bin/main --model ${model_q4_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log + (time ./bin/main --model ${model_q5_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log + (time ./bin/main --model ${model_q6_k} -t 1 -ngl 999 -s 1234 -n 256 --ignore-eos -p "I believe the meaning of life is" ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log + + (time ./bin/perplexity --model ${model_f16} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-f16.log + (time ./bin/perplexity --model ${model_q8_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q8_0.log + (time ./bin/perplexity --model ${model_q4_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_0.log + (time ./bin/perplexity --model ${model_q4_1} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_1.log + (time ./bin/perplexity --model ${model_q5_0} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_0.log + (time ./bin/perplexity --model ${model_q5_1} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_1.log + (time ./bin/perplexity --model ${model_q2_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q2_k.log + (time ./bin/perplexity --model ${model_q3_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q3_k.log + (time ./bin/perplexity --model ${model_q4_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q4_k.log + (time ./bin/perplexity --model ${model_q5_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q5_k.log + (time ./bin/perplexity --model ${model_q6_k} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-tg-q6_k.log + + (time ./bin/imatrix --model ${model_f16} -f ${wiki_test} -t 1 -ngl 999 -c 2048 -b 512 --chunks 4 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log + + (time ./bin/save-load-state -ngl 10 --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/save-load-state -fa -ngl 10 --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/save-load-state -ngl 99 --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/save-load-state -fa -ngl 99 --model ${model_q4_0} ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + + function check_ppl { + qnt="$1" + ppl=$(echo "$2" | grep -oE "[0-9]+\.[0-9]+" | tail -n 1) + + if [ $(echo "$ppl > 20.0" | bc) -eq 1 ]; then + printf ' - %s @ %s (FAIL: ppl > 20.0)\n' "$qnt" "$ppl" + return 20 + fi + + printf ' - %s @ %s OK\n' "$qnt" "$ppl" + return 0 + } + + check_ppl "f16" "$(cat $OUT/${ci}-tg-f16.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q8_0" "$(cat $OUT/${ci}-tg-q8_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q4_0" "$(cat $OUT/${ci}-tg-q4_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q4_1" "$(cat $OUT/${ci}-tg-q4_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q5_0" "$(cat $OUT/${ci}-tg-q5_0.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q5_1" "$(cat $OUT/${ci}-tg-q5_1.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + #check_ppl "q2_k" "$(cat $OUT/${ci}-tg-q2_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log # note: ppl > 20.0 for this quant and model + check_ppl "q3_k" "$(cat $OUT/${ci}-tg-q3_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q4_k" "$(cat $OUT/${ci}-tg-q4_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q5_k" "$(cat $OUT/${ci}-tg-q5_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + check_ppl "q6_k" "$(cat $OUT/${ci}-tg-q6_k.log | grep "^\[1\]")" | tee -a $OUT/${ci}-ppl.log + + cat $OUT/${ci}-imatrix.log | grep "Final" >> $OUT/${ci}-imatrix-sum.log + + set +e +} + +function gg_sum_pythia_2_8b { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Pythia 2.8B:\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '- perplexity:\n%s\n' "$(cat $OUT/${ci}-ppl.log)" + gg_printf '- imatrix:\n```\n%s\n```\n' "$(cat $OUT/${ci}-imatrix-sum.log)" + gg_printf '- f16: \n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-f16.log)" + gg_printf '- q8_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q8_0.log)" + gg_printf '- q4_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_0.log)" + gg_printf '- q4_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_1.log)" + gg_printf '- q5_0:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_0.log)" + gg_printf '- q5_1:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_1.log)" + gg_printf '- q2_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q2_k.log)" + gg_printf '- q3_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q3_k.log)" + gg_printf '- q4_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q4_k.log)" + gg_printf '- q5_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q5_k.log)" + gg_printf '- q6_k:\n```\n%s\n```\n' "$(cat $OUT/${ci}-tg-q6_k.log)" + gg_printf '- save-load-state: \n```\n%s\n```\n' "$(cat $OUT/${ci}-save-load-state.log)" +} + # bge-small function gg_run_embd_bge_small { @@ -552,7 +688,7 @@ function gg_run_embd_bge_small { (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log (time make -j ) 2>&1 | tee -a $OUT/${ci}-make.log - python3 ../convert-hf-to-gguf.py ${path_models} + python3 ../convert-hf-to-gguf.py ${path_models} --outfile ${path_models}/ggml-model-f16.gguf model_f16="${path_models}/ggml-model-f16.gguf" model_q8_0="${path_models}/ggml-model-q8_0.gguf" @@ -606,10 +742,10 @@ if [ -z ${GG_BUILD_LOW_PERF} ]; then if [ -z ${GG_BUILD_VRAM_GB} ] || [ ${GG_BUILD_VRAM_GB} -ge 8 ]; then if [ -z ${GG_BUILD_CUDA} ]; then - #test $ret -eq 0 && gg_run open_llama_3b_v2 - date # dummy + test $ret -eq 0 && gg_run pythia_1_4b else - test $ret -eq 0 && gg_run open_llama_7b_v2 + test $ret -eq 0 && gg_run pythia_2_8b + #test $ret -eq 0 && gg_run open_llama_7b_v2 fi test $ret -eq 0 && gg_run ctest_with_model_debug test $ret -eq 0 && gg_run ctest_with_model_release diff --git a/llama.cpp b/llama.cpp index 5ff186a57..1f9e10eed 100644 --- a/llama.cpp +++ b/llama.cpp @@ -6718,7 +6718,7 @@ static struct ggml_tensor * llm_build_kqv( cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias); - if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) { + if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX) { ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); } @@ -6727,7 +6727,7 @@ static struct ggml_tensor * llm_build_kqv( struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); cb(kq, "kq", il); - if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3) { + if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX) { // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847 ggml_mul_mat_set_prec(kq, GGML_PREC_F32); From 3015851c5ac7334fb544a23a70a284c117b87044 Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Thu, 23 May 2024 14:29:26 +0200 Subject: [PATCH 121/137] llama : add getters for n_threads/n_threads_batch (#7464) * llama : add getters for n_threads/n_threads_batch This commit adds two new functions to the llama API. The functions can be used to get the number of threads used for generating a single token and the number of threads used for prompt and batch processing (multiple tokens). The motivation for this is that we want to be able to get the number of threads that the a context is using. The main use case is for a testing/verification that the number of threads is set correctly. Signed-off-by: Daniel Bevenius * squash! llama : add getters for n_threads/n_threads_batch Rename the getters to llama_n_threads and llama_n_threads_batch. Signed-off-by: Daniel Bevenius --------- Signed-off-by: Daniel Bevenius --- llama.cpp | 8 ++++++++ llama.h | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/llama.cpp b/llama.cpp index 1f9e10eed..e540c1b39 100644 --- a/llama.cpp +++ b/llama.cpp @@ -17410,6 +17410,14 @@ void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_ ctx->cparams.n_threads_batch = n_threads_batch; } +uint32_t llama_n_threads(struct llama_context * ctx) { + return ctx->cparams.n_threads; +} + +uint32_t llama_n_threads_batch(struct llama_context * ctx) { + return ctx->cparams.n_threads_batch; +} + void llama_set_abort_callback(struct llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) { ctx->abort_callback = abort_callback; ctx->abort_callback_data = abort_callback_data; diff --git a/llama.h b/llama.h index b7bf2afcb..16cece5db 100644 --- a/llama.h +++ b/llama.h @@ -759,6 +759,12 @@ extern "C" { // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch); + // Get the number of threads used for generation of a single token. + LLAMA_API uint32_t llama_n_threads(struct llama_context * ctx); + + // Get the number of threads used for prompt and batch processing (multiple token). + LLAMA_API uint32_t llama_n_threads_batch(struct llama_context * ctx); + // Set whether to use causal attention or not // If set to true, the model will only attend to the past tokens LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); From 8b94e799dfa482adf63419df4905dc79b37e179f Mon Sep 17 00:00:00 2001 From: Raj Hammeer Singh Hada Date: Thu, 23 May 2024 18:00:13 +0530 Subject: [PATCH 122/137] readme : add Bunny in supported models [no ci] (#7469) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ccd12e211..461259afb 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,7 @@ Typically finetunes of the base models below are supported as well. - [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL) - [x] [Mini CPM](https://huggingface.co/models?search=MiniCPM) - [x] [Moondream](https://huggingface.co/vikhyatk/moondream2) +- [x] [Bunny](https://github.com/BAAI-DCAI/Bunny) **HTTP server** From 007489e895bad02e4e54758bf0bdf2d6a4cdb7c1 Mon Sep 17 00:00:00 2001 From: Tristan Druyen Date: Thu, 23 May 2024 16:15:15 +0200 Subject: [PATCH 123/137] Fix phi3 chat template confusion with zephyr (#7449) * Fix phi3 template matching vs zephyr * Add regression test for new phi3 chat template * Implement review suggestions * Fix phi3 jinja test templates & match by <|end|> * Apply suggestion Co-authored-by: Xuan Son Nguyen * Add all phi3 template variants in tests * Remove unneeded message trimming Co-authored-by: Xuan Son Nguyen * Fix tests to not expect trimmed messages --------- Co-authored-by: Xuan Son Nguyen --- llama.cpp | 18 +++++++++--------- tests/test-chat-template.cpp | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/llama.cpp b/llama.cpp index e540c1b39..15c660775 100644 --- a/llama.cpp +++ b/llama.cpp @@ -17852,6 +17852,15 @@ static int32_t llama_chat_apply_template_internal( } } // llama2 templates seem to not care about "add_generation_prompt" + } else if (tmpl == "phi3" || (tmpl.find("<|assistant|>") != std::string::npos && tmpl.find("<|end|>") != std::string::npos)) { + // Phi 3 + for (auto message : chat) { + std::string role(message->role); + ss << "<|" << role << "|>\n" << message->content << "<|end|>\n"; + } + if (add_ass) { + ss << "<|assistant|>\n"; + } } else if (tmpl == "zephyr" || tmpl.find("<|user|>") != std::string::npos) { // zephyr template for (auto message : chat) { @@ -17984,15 +17993,6 @@ static int32_t llama_chat_apply_template_internal( if (add_ass) { ss << "<|start_header_id|>assistant<|end_header_id|>\n\n"; } - } else if (tmpl == "phi3" || (tmpl.find("<|assistant|>") != std::string::npos && tmpl.find("<|end|>") != std::string::npos )) { - // Phi 3 - for (auto message : chat) { - std::string role(message->role); - ss << "<|" << role << "|>\n" << trim(message->content) << "<|end|>\n"; - } - if (add_ass) { - ss << "<|assistant|>\n"; - } } else { // template not supported return -1; diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index 4fe9183b9..cef9a650b 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -49,8 +49,14 @@ int main(void) { "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true %}{% set loop_messages = messages %}{% set system_message = 'You are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by providing thorough responses. You are trained by Cohere.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% if system_message != false %}{{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% elif message['role'] == 'assistant' %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}{% endif %}", // Llama-3 "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}", - // Phi-3 - "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + ' ' + message['content'] + '<|end|> ' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|> ' }}{% else %}{{ eos_token }}{% endif %}" + //Phi-3-mini + "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}", + //Phi-3-small + "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}", + //Phi-3-medium + "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}", + //Phi-3-vision + "{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{- '<|assistant|>\n' -}}{% endif %}" }; std::vector expected_output = { // teknium/OpenHermes-2.5-Mistral-7B @@ -79,8 +85,14 @@ int main(void) { "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>You are a helpful assistant<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>Hi there<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Who are you<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>I am an assistant<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Another question<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", // Llama 3 "<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nHi there<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWho are you<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nI am an assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nAnother question<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", - // Phi 3 - "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\nI am an assistant<|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n", + //Phi-3-mini + "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n", + //Phi-3-small + "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n", + //Phi-3-medium + "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n", + //Phi-3-vision + "<|system|>\nYou are a helpful assistant<|end|>\n<|user|>\nHello<|end|>\n<|assistant|>\nHi there<|end|>\n<|user|>\nWho are you<|end|>\n<|assistant|>\n I am an assistant <|end|>\n<|user|>\nAnother question<|end|>\n<|assistant|>\n", }; std::vector formatted_chat(1024); int32_t res; From 1debe72737ea131cb52975da3d53ed3a835df3a6 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 23 May 2024 17:17:43 +0300 Subject: [PATCH 124/137] ggml : silence UB sanitizer error during iq2_xxs quantization (#0) --- ggml-quants.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml-quants.c b/ggml-quants.c index 88f58a339..bb01ce93c 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -12144,7 +12144,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict printf("\n"); GGML_ASSERT(false); } - q2[2*ib+0] |= (grid_index << 8*k); + q2[2*ib+0] |= ((uint32_t) grid_index << 8*k); q2[2*ib+1] |= (block_signs[k] << 7*k); } GGML_ASSERT(scale >= 0); From 74f33adf5f8b20b08fc5a6aa17ce081abe86ef2f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 23 May 2024 17:43:18 +0300 Subject: [PATCH 125/137] readme : remove trailing space (#7469) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 461259afb..2ee267fdf 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Typically finetunes of the base models below are supported as well. - [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL) - [x] [Mini CPM](https://huggingface.co/models?search=MiniCPM) - [x] [Moondream](https://huggingface.co/vikhyatk/moondream2) -- [x] [Bunny](https://github.com/BAAI-DCAI/Bunny) +- [x] [Bunny](https://github.com/BAAI-DCAI/Bunny) **HTTP server** From 0df0aa8e43c3378975269a51f9b876c8692e70da Mon Sep 17 00:00:00 2001 From: Neo Zhang <14088817+arthw@users.noreply.github.com> Date: Fri, 24 May 2024 10:06:56 +0800 Subject: [PATCH 126/137] add build shared lib in win release package (#7438) --- examples/sycl/win-build-sycl.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/sycl/win-build-sycl.bat b/examples/sycl/win-build-sycl.bat index 1b0dc41ba..b8037aae8 100644 --- a/examples/sycl/win-build-sycl.bat +++ b/examples/sycl/win-build-sycl.bat @@ -13,10 +13,10 @@ if %errorlevel% neq 0 goto ERROR :: for FP16 :: faster for long-prompt inference -:: cmake -G "MinGW Makefiles" .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DCMAKE_BUILD_TYPE=Release -DLLAMA_SYCL_F16=ON +:: cmake -G "MinGW Makefiles" .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DLLAMA_SYCL_F16=ON :: for FP32 -cmake -G "MinGW Makefiles" .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DCMAKE_BUILD_TYPE=Release +cmake -G "MinGW Makefiles" .. -DLLAMA_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icx -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release if %errorlevel% neq 0 goto ERROR :: build example/main only :: make main From fbca2f27fc7fa9aa4a8ad0357478fdb908472908 Mon Sep 17 00:00:00 2001 From: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Date: Fri, 24 May 2024 14:31:13 +0200 Subject: [PATCH 127/137] Add support for ArcticForCausalLM (#7020) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * common : increase max number of experts to 128 * common : add tensor LLM_TENSOR_FFN_NORM_EXPS for normalization before MoE that runs in parallel to attention + ffn * gguf-py : add architecture-specific block mappings that override selected general block mappings * convert-hf : add model conversion support for ArcticForCausalLM * convert-hf : use added_tokens_decoder from tokenizer_config.json to redefine tokens from SentencePiece model (only for ArcticForCausalLM) * llama : add inference support for LLM_ARCH_ARCTIC --------- Co-authored-by: Stanisław Szymczyk --- convert-hf-to-gguf.py | 151 ++++++++++++++++ gguf-py/gguf/constants.py | 25 +++ gguf-py/gguf/tensor_mapping.py | 19 ++- llama.cpp | 304 ++++++++++++++++++++++++++++----- 4 files changed, 456 insertions(+), 43 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 5a00a5e89..998877c26 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -2466,6 +2466,157 @@ class JinaBertV2Model(BertModel): self.gguf_writer.add_add_eos_token(True) +@Model.register("ArcticForCausalLM") +class ArcticModel(Model): + model_arch = gguf.MODEL_ARCH.ARCTIC + + def set_vocab(self): + # The reason for using a custom implementation here is that the + # snowflake-arctic-instruct model redefined tokens 31998 and 31999 from + # tokenizer.model and used them as BOS and EOS instead of adding new tokens. + from sentencepiece import SentencePieceProcessor + + tokenizer_path = self.dir_model / 'tokenizer.model' + + if not tokenizer_path.is_file(): + logger.error(f'Error: Missing {tokenizer_path}') + sys.exit(1) + + # Read the whole vocabulary from the tokenizer.model file + tokenizer = SentencePieceProcessor() + tokenizer.LoadFromFile(str(tokenizer_path)) + + vocab_size = self.hparams.get('vocab_size', tokenizer.vocab_size()) + + tokens: list[bytes] = [f"[PAD{i}]".encode("utf-8") for i in range(vocab_size)] + scores: list[float] = [-10000.0] * vocab_size + toktypes: list[int] = [SentencePieceTokenTypes.UNKNOWN] * vocab_size + + for token_id in range(tokenizer.vocab_size()): + + piece = tokenizer.IdToPiece(token_id) + text = piece.encode("utf-8") + score = tokenizer.GetScore(token_id) + + toktype = SentencePieceTokenTypes.NORMAL + if tokenizer.IsUnknown(token_id): + toktype = SentencePieceTokenTypes.UNKNOWN + elif tokenizer.IsControl(token_id): + toktype = SentencePieceTokenTypes.CONTROL + elif tokenizer.IsUnused(token_id): + toktype = SentencePieceTokenTypes.UNUSED + elif tokenizer.IsByte(token_id): + toktype = SentencePieceTokenTypes.BYTE + + tokens[token_id] = text + scores[token_id] = score + toktypes[token_id] = toktype + + # Use the added_tokens_decoder field from tokeniser_config.json as the source + # of information about added/redefined tokens and modify them accordingly. + tokenizer_config_file = self.dir_model / 'tokenizer_config.json' + if tokenizer_config_file.is_file(): + with open(tokenizer_config_file, "r", encoding="utf-8") as f: + tokenizer_config_json = json.load(f) + + if "added_tokens_decoder" in tokenizer_config_json: + added_tokens_decoder = tokenizer_config_json["added_tokens_decoder"] + for token_id, token_json in added_tokens_decoder.items(): + token_id = int(token_id) + if (token_id >= vocab_size): + logger.debug(f'ignore token {token_id}: id is out of range, max={vocab_size - 1}') + continue + + token_content = token_json["content"] + token_type = SentencePieceTokenTypes.USER_DEFINED + token_score = -10000.0 + + # Map unk_token to UNKNOWN, other special tokens to CONTROL + # Set the score to 0.0 as in the original tokenizer.model + if ("special" in token_json) and token_json["special"]: + if token_content == tokenizer_config_json["unk_token"]: + token_type = SentencePieceTokenTypes.UNKNOWN + else: + token_type = SentencePieceTokenTypes.CONTROL + token_score = 0.0 + + logger.info(f"Setting added token {token_id} to '{token_content}' (type: {token_type}, score: {token_score:.2f})") + tokens[token_id] = token_content.encode("utf-8") + toktypes[token_id] = token_type + scores[token_id] = token_score + + self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) + special_vocab.add_to_gguf(self.gguf_writer) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hparams = self.hparams + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_rope_dimension_count(hparams["hidden_size"] // hparams["num_attention_heads"]) + + _experts: list[dict[str, Tensor]] | None = None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + n_head = self.hparams["num_attention_heads"] + n_kv_head = self.hparams.get("num_key_value_heads") + + if name.endswith("q_proj.weight"): + data_torch = LlamaModel.permute(data_torch, n_head, n_head) + if name.endswith("k_proj.weight"): + data_torch = LlamaModel.permute(data_torch, n_head, n_kv_head) + + # process the experts separately + if name.find("block_sparse_moe.experts") != -1: + n_experts = self.hparams["num_local_experts"] + + assert bid is not None + + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + + self._experts[bid][name] = data_torch + + if len(self._experts[bid]) >= n_experts * 3: + tensors: list[tuple[str, Tensor]] = [] + + # merge the experts into a single 3d tensor + for wid in ["w1", "w2", "w3"]: + datas: list[Tensor] = [] + + for xid in range(n_experts): + ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight" + datas.append(self._experts[bid][ename]) + del self._experts[bid][ename] + + data_torch = torch.stack(datas, dim=0) + + merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight" + + new_name = self.map_tensor_name(merged_name) + + tensors.append((new_name, data_torch)) + return tensors + else: + return [] + + return [(self.map_tensor_name(name), data_torch)] + + def write_tensors(self): + super().write_tensors() + + if self._experts is not None: + # flatten `list[dict[str, Tensor]]` into `list[str]` + experts = [k for d in self._experts for k in d.keys()] + if len(experts) > 0: + raise ValueError(f"Unprocessed experts: {experts}") + + ###### CONVERSION LOGIC ###### diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 67e23dcc1..c9ae259e1 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -139,6 +139,7 @@ class MODEL_ARCH(IntEnum): COMMAND_R = auto() DBRX = auto() OLMO = auto() + ARCTIC = auto() class MODEL_TENSOR(IntEnum): @@ -167,6 +168,7 @@ class MODEL_TENSOR(IntEnum): FFN_DOWN = auto() FFN_UP = auto() FFN_ACT = auto() + FFN_NORM_EXP = auto() FFN_GATE_EXP = auto() FFN_DOWN_EXP = auto() FFN_UP_EXP = auto() @@ -218,6 +220,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.COMMAND_R: "command-r", MODEL_ARCH.DBRX: "dbrx", MODEL_ARCH.OLMO: "olmo", + MODEL_ARCH.ARCTIC: "arctic", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -251,6 +254,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.FFN_DOWN_SHEXP: "blk.{bid}.ffn_down_shexp", MODEL_TENSOR.FFN_UP_SHEXP: "blk.{bid}.ffn_up_shexp", MODEL_TENSOR.FFN_ACT: "blk.{bid}.ffn", + MODEL_TENSOR.FFN_NORM_EXP: "blk.{bid}.ffn_norm_exps", MODEL_TENSOR.FFN_GATE_EXP: "blk.{bid}.ffn_gate_exps", MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down_exps", MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps", @@ -732,6 +736,27 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.ARCTIC: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_NORM_EXP, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + ], # TODO } diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 8e1cac915..8b1b21d78 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -244,6 +244,7 @@ class TensorNameMap: "encoder.layers.{bid}.mlp.fc11", # nomic-bert "model.layers.{bid}.mlp.c_fc", # starcoder2 "encoder.layer.{bid}.mlp.gated_layers_v", # jina-bert-v2 + "model.layers.{bid}.residual_mlp.w3", # arctic ), MODEL_TENSOR.FFN_UP_EXP: ( @@ -272,6 +273,7 @@ class TensorNameMap: "encoder.layers.{bid}.mlp.fc12", # nomic-bert "encoder.layer.{bid}.mlp.gated_layers_w", # jina-bert-v2 "transformer.h.{bid}.mlp.linear_1", # refact + "model.layers.{bid}.residual_mlp.w1", # arctic ), MODEL_TENSOR.FFN_GATE_EXP: ( @@ -306,6 +308,7 @@ class TensorNameMap: "encoder.layers.{bid}.mlp.fc2", # nomic-bert "model.layers.{bid}.mlp.c_proj", # starcoder2 "encoder.layer.{bid}.mlp.wo", # jina-bert-v2 + "model.layers.{bid}.residual_mlp.w2", # arctic ), MODEL_TENSOR.FFN_DOWN_EXP: ( @@ -382,6 +385,18 @@ class TensorNameMap: ), } + # architecture-specific block mappings + arch_block_mappings_cfg: dict[MODEL_ARCH, dict[MODEL_TENSOR, tuple[str, ...]]] = { + MODEL_ARCH.ARCTIC: { + MODEL_TENSOR.FFN_NORM: ( + "model.layers.{bid}.residual_layernorm", + ), + MODEL_TENSOR.FFN_NORM_EXP: ( + "model.layers.{bid}.post_attention_layernorm", + ), + }, + } + mapping: dict[str, tuple[MODEL_TENSOR, str]] def __init__(self, arch: MODEL_ARCH, n_blocks: int): @@ -393,12 +408,14 @@ class TensorNameMap: self.mapping[tensor_name] = (tensor, tensor_name) for key in keys: self.mapping[key] = (tensor, tensor_name) + if arch in self.arch_block_mappings_cfg: + self.block_mappings_cfg.update(self.arch_block_mappings_cfg[arch]) for bid in range(n_blocks): for tensor, keys in self.block_mappings_cfg.items(): if tensor not in MODEL_TENSORS[arch]: continue # TODO: make this configurable - n_experts = 60 + n_experts = 128 for xid in range(n_experts): tensor_name = TENSOR_NAMES[tensor].format(bid = bid, xid = xid) self.mapping[tensor_name] = (tensor, tensor_name) diff --git a/llama.cpp b/llama.cpp index 15c660775..3c9fe15bb 100644 --- a/llama.cpp +++ b/llama.cpp @@ -103,7 +103,7 @@ #endif #define LLAMA_MAX_NODES 8192 -#define LLAMA_MAX_EXPERTS 60 +#define LLAMA_MAX_EXPERTS 128 // // logging @@ -221,6 +221,7 @@ enum llm_arch { LLM_ARCH_COMMAND_R, LLM_ARCH_DBRX, LLM_ARCH_OLMO, + LLM_ARCH_ARCTIC, LLM_ARCH_UNKNOWN, }; @@ -257,6 +258,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_COMMAND_R, "command-r" }, { LLM_ARCH_DBRX, "dbrx" }, { LLM_ARCH_OLMO, "olmo" }, + { LLM_ARCH_ARCTIC, "arctic" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; @@ -455,6 +457,7 @@ enum llm_tensor { LLM_TENSOR_FFN_DOWN_EXP, // split experts for backward compatibility LLM_TENSOR_FFN_GATE_EXP, LLM_TENSOR_FFN_UP_EXP, + LLM_TENSOR_FFN_NORM_EXPS, LLM_TENSOR_FFN_DOWN_EXPS, // merged experts LLM_TENSOR_FFN_GATE_EXPS, LLM_TENSOR_FFN_UP_EXPS, @@ -1032,6 +1035,28 @@ static const std::map> LLM_TENSOR_NA { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, }, }, + { + LLM_ARCH_ARCTIC, + { + { LLM_TENSOR_TOKEN_EMBD, "token_embd" }, + { LLM_TENSOR_OUTPUT_NORM, "output_norm" }, + { LLM_TENSOR_OUTPUT, "output" }, + { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" }, + { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" }, + { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" }, + { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" }, + { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" }, + { LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" }, + { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" }, + { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" }, + { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" }, + { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" }, + { LLM_TENSOR_FFN_NORM_EXPS, "blk.%d.ffn_norm_exps" }, + { LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" }, + { LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" }, + { LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" }, + }, + }, { LLM_ARCH_UNKNOWN, { @@ -1732,6 +1757,7 @@ enum e_model { MODEL_8x7B, MODEL_8x22B, MODEL_16x12B, + MODEL_10B_128x3_66B, }; static const size_t kiB = 1024; @@ -1907,6 +1933,7 @@ struct llama_layer { struct ggml_tensor * ffn_norm_b; struct ggml_tensor * layer_out_norm; struct ggml_tensor * layer_out_norm_b; + struct ggml_tensor * ffn_norm_exps; // ff struct ggml_tensor * ffn_gate; // w1 @@ -3781,47 +3808,48 @@ static std::string llama_model_ftype_name(llama_ftype ftype) { static const char * llama_model_type_name(e_model type) { switch (type) { - case MODEL_14M: return "14M"; - case MODEL_17M: return "17M"; - case MODEL_22M: return "22M"; - case MODEL_33M: return "33M"; - case MODEL_70M: return "70M"; - case MODEL_109M: return "109M"; - case MODEL_137M: return "137M"; - case MODEL_160M: return "160M"; - case MODEL_335M: return "335M"; - case MODEL_410M: return "410M"; - case MODEL_0_5B: return "0.5B"; - case MODEL_1B: return "1B"; - case MODEL_1_4B: return "1.4B"; - case MODEL_2B: return "2B"; - case MODEL_2_8B: return "2.8B"; - case MODEL_3B: return "3B"; - case MODEL_4B: return "4B"; - case MODEL_6_9B: return "6.9B"; - case MODEL_7B: return "7B"; - case MODEL_8B: return "8B"; - case MODEL_12B: return "12B"; - case MODEL_13B: return "13B"; - case MODEL_14B: return "14B"; - case MODEL_15B: return "15B"; - case MODEL_20B: return "20B"; - case MODEL_30B: return "30B"; - case MODEL_34B: return "34B"; - case MODEL_35B: return "35B"; - case MODEL_40B: return "40B"; - case MODEL_65B: return "65B"; - case MODEL_70B: return "70B"; - case MODEL_314B: return "314B"; - case MODEL_SMALL: return "0.1B"; - case MODEL_MEDIUM: return "0.4B"; - case MODEL_LARGE: return "0.8B"; - case MODEL_XL: return "1.5B"; - case MODEL_A2_7B: return "A2.7B"; - case MODEL_8x7B: return "8x7B"; - case MODEL_8x22B: return "8x22B"; - case MODEL_16x12B: return "16x12B"; - default: return "?B"; + case MODEL_14M: return "14M"; + case MODEL_17M: return "17M"; + case MODEL_22M: return "22M"; + case MODEL_33M: return "33M"; + case MODEL_70M: return "70M"; + case MODEL_109M: return "109M"; + case MODEL_137M: return "137M"; + case MODEL_160M: return "160M"; + case MODEL_335M: return "335M"; + case MODEL_410M: return "410M"; + case MODEL_0_5B: return "0.5B"; + case MODEL_1B: return "1B"; + case MODEL_1_4B: return "1.4B"; + case MODEL_2B: return "2B"; + case MODEL_2_8B: return "2.8B"; + case MODEL_3B: return "3B"; + case MODEL_4B: return "4B"; + case MODEL_6_9B: return "6.9B"; + case MODEL_7B: return "7B"; + case MODEL_8B: return "8B"; + case MODEL_12B: return "12B"; + case MODEL_13B: return "13B"; + case MODEL_14B: return "14B"; + case MODEL_15B: return "15B"; + case MODEL_20B: return "20B"; + case MODEL_30B: return "30B"; + case MODEL_34B: return "34B"; + case MODEL_35B: return "35B"; + case MODEL_40B: return "40B"; + case MODEL_65B: return "65B"; + case MODEL_70B: return "70B"; + case MODEL_314B: return "314B"; + case MODEL_SMALL: return "0.1B"; + case MODEL_MEDIUM: return "0.4B"; + case MODEL_LARGE: return "0.8B"; + case MODEL_XL: return "1.5B"; + case MODEL_A2_7B: return "A2.7B"; + case MODEL_8x7B: return "8x7B"; + case MODEL_8x22B: return "8x22B"; + case MODEL_16x12B: return "16x12B"; + case MODEL_10B_128x3_66B: return "10B+128x3.66B"; + default: return "?B"; } } @@ -4343,6 +4371,19 @@ static void llm_load_hparams( default: model.type = e_model::MODEL_UNKNOWN; } } break; + case LLM_ARCH_ARCTIC: + { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + if (hparams.n_expert == 128) { + switch (hparams.n_layer) { + case 35: model.type = e_model::MODEL_10B_128x3_66B; break; + default: model.type = e_model::MODEL_UNKNOWN; + } + } else { + model.type = e_model::MODEL_UNKNOWN; + } + } break; default: (void)0; } @@ -6129,6 +6170,46 @@ static bool llm_load_tensors( layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}); } } break; + case LLM_ARCH_ARCTIC: + { + model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}); + + // output + { + model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); + // if output is NULL, init from the input tok embed + if (model.output == NULL) { + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); + } + } + + for (int i = 0; i < n_layer; ++i) { + ggml_context * ctx_layer = ctx_for_layer(i); + ggml_context * ctx_split = ctx_for_layer_split(i); + + auto & layer = model.layers[i]; + + layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}); + + layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}); + layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}); + layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}); + layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}); + + layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}); + + layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_embd}); + layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_embd, n_embd}); + layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_embd}); + + layer.ffn_gate_inp = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}); + layer.ffn_norm_exps = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM_EXPS, "weight", i), {n_embd}); + layer.ffn_gate_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, false); + layer.ffn_down_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), { n_ff, n_embd, n_expert}); + layer.ffn_up_exps = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}); + } + } break; default: throw std::runtime_error("unknown architecture"); } @@ -10790,6 +10871,140 @@ struct llm_build_context { return gf; } + + struct ggml_cgraph * build_arctic() { + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false); + + // mutable variable, needed during the last layer of the computation to skip unused tokens + int32_t n_tokens = this->n_tokens; + + const int64_t n_embd_head = hparams.n_embd_head_v; + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k); + GGML_ASSERT(n_embd_head == hparams.n_rot); + + struct ggml_tensor * cur; + struct ggml_tensor * inpL; + + inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb); + + // inp_pos - contains the positions + struct ggml_tensor * inp_pos = build_inp_pos(); + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = build_inp_KQ_mask(); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * inpSA = inpL; + + // norm + cur = llm_build_norm(ctx0, inpL, hparams, + model.layers[il].attn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "attn_norm", il); + + // self-attention + { + // compute Q and K and RoPE them + struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur); + cb(Qcur, "Qcur", il); + + struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur); + cb(Kcur, "Kcur", il); + + struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur); + cb(Vcur, "Vcur", il); + + Qcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Qcur, "Qcur", il); + + Kcur = ggml_rope_ext( + ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr, + n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow + ); + cb(Kcur, "Kcur", il); + + cur = llm_build_kv(ctx0, model, hparams, cparams, kv_self, gf, + model.layers[il].wo, NULL, + Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il); + } + + if (il == n_layer - 1) { + // skip computing output for unused tokens + struct ggml_tensor * inp_out_ids = build_inp_out_ids(); + n_tokens = n_outputs; + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + // feed-forward network + cur = llm_build_norm(ctx0, ffn_inp, hparams, + model.layers[il].ffn_norm, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm", il); + + cur = llm_build_ffn(ctx0, cur, + model.layers[il].ffn_up, NULL, + model.layers[il].ffn_gate, NULL, + model.layers[il].ffn_down, NULL, + NULL, + LLM_FFN_SILU, LLM_FFN_PAR, cb, il); + cb(cur, "ffn_out", il); + + struct ggml_tensor * ffn_out = ggml_add(ctx0, cur, ffn_inp); + cb(ffn_out, "ffn_out", il); + + // MoE + cur = llm_build_norm(ctx0, inpSA, hparams, + model.layers[il].ffn_norm_exps, NULL, + LLM_NORM_RMS, cb, il); + cb(cur, "ffn_norm_exps", il); + + cur = llm_build_moe_ffn(ctx0, cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + n_expert, n_expert_used, + LLM_FFN_SILU, true, + cb, il); + cb(cur, "ffn_moe_out", il); + + cur = ggml_add(ctx0, cur, ffn_out); + cb(cur, "ffn_out", il); + + ggml_tensor * layer_dir = lctx.cvec.tensor_for(il); + if (layer_dir != nullptr) { + cur = ggml_add(ctx0, cur, layer_dir); + } + cb(cur, "l_out", il); + + // input for next layer + inpL = cur; + } + + cur = inpL; + + cur = llm_build_norm(ctx0, cur, hparams, + model.output_norm, NULL, + LLM_NORM_RMS, cb, -1); + cb(cur, "result_norm", -1); + + // lm_head + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; + } }; static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector & ids) { @@ -11004,6 +11219,10 @@ static struct ggml_cgraph * llama_build_graph( { result = llm.build_gptneox(); } break; + case LLM_ARCH_ARCTIC: + { + result = llm.build_arctic(); + } break; default: GGML_ASSERT(false); } @@ -16015,6 +16234,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) { case LLM_ARCH_XVERSE: case LLM_ARCH_COMMAND_R: case LLM_ARCH_OLMO: + case LLM_ARCH_ARCTIC: return LLAMA_ROPE_TYPE_NORM; // the pairs of head values are offset by n_rot/2 From 27891f6db03de6e3fd5941983838c29bef253352 Mon Sep 17 00:00:00 2001 From: Brian Date: Fri, 24 May 2024 23:47:56 +1000 Subject: [PATCH 128/137] docker.yml: disable light-intel and server-intel test (#7515) * docker.yml: disable light-intel test * docker.yml: disable server-intel test --- .github/workflows/docker.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9b03d19bc..c2838cbd9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -42,8 +42,9 @@ jobs: - { tag: "light-rocm", dockerfile: ".devops/main-rocm.Dockerfile", platforms: "linux/amd64,linux/arm64" } - { tag: "full-rocm", dockerfile: ".devops/full-rocm.Dockerfile", platforms: "linux/amd64,linux/arm64" } - { tag: "server-rocm", dockerfile: ".devops/server-rocm.Dockerfile", platforms: "linux/amd64,linux/arm64" } - - { tag: "light-intel", dockerfile: ".devops/main-intel.Dockerfile", platforms: "linux/amd64" } - - { tag: "server-intel", dockerfile: ".devops/server-intel.Dockerfile", platforms: "linux/amd64" } + # TODO: Disabled due to build issues https://github.com/ggerganov/llama.cpp/issues/7507 + #- { tag: "light-intel", dockerfile: ".devops/main-intel.Dockerfile", platforms: "linux/amd64" } + #- { tag: "server-intel", dockerfile: ".devops/server-intel.Dockerfile", platforms: "linux/amd64" } steps: - name: Check out the repo uses: actions/checkout@v4 From d041d2ceaaf50e058622d92921b3e680ffa4e9e7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 24 May 2024 18:59:06 +0300 Subject: [PATCH 129/137] flake.lock: Update (#7232) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/e5d10a24b66c3ea8f150e47dfdb0416ab7c3390e?narHash=sha256-yzcRNDoyVP7%2BSCNX0wmuDju1NUCt8Dz9%2BlyUXEI0dbI%3D' (2024-05-02) → 'github:hercules-ci/flake-parts/8dc45382d5206bd292f9c2768b8058a8fd8311d9?narHash=sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78%3D' (2024-05-16) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/63c3a29ca82437c87573e4c6919b09a24ea61b0f?narHash=sha256-4cPymbty65RvF1DWQfc%2BBc8B233A1BWxJnNULJKQ1EY%3D' (2024-05-02) → 'github:NixOS/nixpkgs/4a6b83b05df1a8bd7d99095ec4b4d271f2956b64?narHash=sha256-%2BNpbZRCRisUHKQJZF3CT%2Bxn14ZZQO%2BKjxIIanH3Pvn4%3D' (2024-05-17) Co-authored-by: github-actions[bot] --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index c9ead0bf7..451dfd32f 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1714641030, - "narHash": "sha256-yzcRNDoyVP7+SCNX0wmuDju1NUCt8Dz9+lyUXEI0dbI=", + "lastModified": 1715865404, + "narHash": "sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "e5d10a24b66c3ea8f150e47dfdb0416ab7c3390e", + "rev": "8dc45382d5206bd292f9c2768b8058a8fd8311d9", "type": "github" }, "original": { @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1714635257, - "narHash": "sha256-4cPymbty65RvF1DWQfc+Bc8B233A1BWxJnNULJKQ1EY=", + "lastModified": 1715961556, + "narHash": "sha256-+NpbZRCRisUHKQJZF3CT+xn14ZZQO+KjxIIanH3Pvn4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "63c3a29ca82437c87573e4c6919b09a24ea61b0f", + "rev": "4a6b83b05df1a8bd7d99095ec4b4d271f2956b64", "type": "github" }, "original": { From b83bab15a5d2a1e7807d09613a9b34309d86cfaa Mon Sep 17 00:00:00 2001 From: compilade Date: Fri, 24 May 2024 21:11:48 -0400 Subject: [PATCH 130/137] gguf-py : fix and simplify quantized shape round-trip (#7483) * gguf-py : fix and simplify quantized shape round-trip * gguf-py : remove unused import --- convert-hf-to-gguf.py | 7 +++---- gguf-py/gguf/gguf_reader.py | 6 +++++- gguf-py/gguf/gguf_writer.py | 8 +++----- gguf-py/gguf/quants.py | 16 +++++++++++++++- gguf-py/scripts/gguf-new-metadata.py | 4 +--- 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py index 998877c26..51549ac72 100755 --- a/convert-hf-to-gguf.py +++ b/convert-hf-to-gguf.py @@ -313,11 +313,10 @@ class Model: data = data.astype(np.float32) data_qtype = gguf.GGMLQuantizationType.F32 - block_size, type_size = gguf.GGML_QUANT_SIZES[data_qtype] + shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape + # reverse shape to make it similar to the internal ggml dimension order - shape_str = f"""{{{', '.join(str(n) for n in reversed( - (*data.shape[:-1], data.shape[-1] * data.dtype.itemsize // type_size * block_size)) - )}}}""" + shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}" # n_dims is implicit in the shape logger.info(f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype} --> {data_qtype.name}, shape = {shape_str}") diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index 21b089f8a..e48bc00c3 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -12,6 +12,8 @@ from typing import Any, Literal, NamedTuple, TypeVar, Union import numpy as np import numpy.typing as npt +from .quants import quant_shape_to_byte_shape + if __name__ == "__main__": import sys from pathlib import Path @@ -251,6 +253,7 @@ class GGUFReader: tensor_names.add(tensor_name) ggml_type = GGMLQuantizationType(raw_dtype[0]) n_elems = int(np.prod(dims)) + np_dims = tuple(reversed(dims.tolist())) block_size, type_size = GGML_QUANT_SIZES[ggml_type] n_bytes = n_elems * type_size // block_size data_offs = int(start_offs + offset_tensor[0]) @@ -279,6 +282,7 @@ class GGUFReader: else: item_count = n_bytes item_type = np.uint8 + np_dims = quant_shape_to_byte_shape(np_dims, ggml_type) tensors.append(ReaderTensor( name = tensor_name, tensor_type = ggml_type, @@ -286,7 +290,7 @@ class GGUFReader: n_elements = n_elems, n_bytes = n_bytes, data_offset = data_offs, - data = self._get(data_offs, item_type, item_count), + data = self._get(data_offs, item_type, item_count).reshape(np_dims), field = field, )) self.tensors = tensors diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 8b41b54ea..c194dd5dd 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -13,7 +13,6 @@ from string import ascii_letters, digits import numpy as np from .constants import ( - GGML_QUANT_SIZES, GGUF_DEFAULT_ALIGNMENT, GGUF_MAGIC, GGUF_VERSION, @@ -26,6 +25,8 @@ from .constants import ( TokenType, ) +from .quants import quant_shape_from_byte_shape + logger = logging.getLogger(__name__) @@ -229,10 +230,7 @@ class GGUFWriter: else: dtype = raw_dtype if tensor_dtype == np.uint8: - block_size, type_size = GGML_QUANT_SIZES[raw_dtype] - if tensor_shape[-1] % type_size != 0: - raise ValueError(f"Quantized tensor row size ({tensor_shape[-1]}) is not a multiple of {dtype.name} type size ({type_size})") - tensor_shape = tuple(tensor_shape[:-1]) + (tensor_shape[-1] // type_size * block_size,) + tensor_shape = quant_shape_from_byte_shape(tensor_shape, raw_dtype) n_dims = len(tensor_shape) self.ti_data += self._pack("I", n_dims) for i in range(n_dims): diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py index e7fc0eae3..b22eec166 100644 --- a/gguf-py/gguf/quants.py +++ b/gguf-py/gguf/quants.py @@ -1,5 +1,5 @@ from __future__ import annotations -from typing import Callable +from typing import Callable, Sequence from numpy.typing import DTypeLike @@ -9,6 +9,20 @@ from .lazy import LazyNumpyTensor import numpy as np +def quant_shape_to_byte_shape(shape: Sequence[int], quant_type: GGMLQuantizationType): + block_size, type_size = GGML_QUANT_SIZES[quant_type] + if shape[-1] % block_size != 0: + raise ValueError(f"Quantized tensor row size ({shape[-1]}) is not a multiple of {quant_type.name} block size ({block_size})") + return (*shape[:-1], shape[-1] // block_size * type_size) + + +def quant_shape_from_byte_shape(shape: Sequence[int], quant_type: GGMLQuantizationType): + block_size, type_size = GGML_QUANT_SIZES[quant_type] + if shape[-1] % type_size != 0: + raise ValueError(f"Quantized tensor bytes per row ({shape[-1]}) is not a multiple of {quant_type.name} type size ({type_size})") + return (*shape[:-1], shape[-1] // type_size * block_size) + + # same as ggml_compute_fp32_to_bf16 in ggml-impl.h def __compute_fp32_to_bf16(n: np.ndarray) -> np.ndarray: n = n.astype(np.float32, copy=False).view(np.int32) diff --git a/gguf-py/scripts/gguf-new-metadata.py b/gguf-py/scripts/gguf-new-metadata.py index 63d3c5d8f..c9f1927f6 100755 --- a/gguf-py/scripts/gguf-new-metadata.py +++ b/gguf-py/scripts/gguf-new-metadata.py @@ -118,9 +118,7 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new for tensor in reader.tensors: total_bytes += tensor.n_bytes - # Dimensions are written in reverse order, so flip them first - shape = np.flipud(tensor.shape).tolist() - writer.add_tensor_info(tensor.name, shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) + writer.add_tensor_info(tensor.name, tensor.data.shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) bar = tqdm(desc="Writing", total=total_bytes, unit="byte", unit_scale=True) From 57684331fc2d685f7d1f5775af0b9e47d1829833 Mon Sep 17 00:00:00 2001 From: Mikko Juola Date: Fri, 24 May 2024 18:14:42 -0700 Subject: [PATCH 131/137] Make tokenize CLI tool have nicer command line arguments. (#6188) * Make tokenizer.cpp CLI tool nicer. Before this commit, tokenize was a simple CLI tool like this: tokenize MODEL_FILENAME PROMPT [--ids] This simple tool loads the model, takes the prompt, and shows the tokens llama.cpp is interpreting. This changeset makes the tokenize more sophisticated, and more useful for debugging and troubleshooting: tokenize [-m, --model MODEL_FILENAME] [--ids] [--stdin] [--prompt] [-f, --file] [--no-bos] [--log-disable] It also behaves nicer on Windows now, interpreting and rendering Unicode from command line arguments and pipes no matter what code page the user has set on their terminal. * style fix: strlen(str) == 0 --> *str == 0 * Simplify tokenize.cpp; by getting rid of handling positional style arguments. It must now be invoked with long --model, --prompt etc. arguments only. Shortens the code. * tokenize.cpp: iostream header no longer required --------- Co-authored-by: Georgi Gerganov Co-authored-by: brian khuu --- examples/tokenize/tokenize.cpp | 368 ++++++++++++++++++++++++++++++++- 1 file changed, 359 insertions(+), 9 deletions(-) diff --git a/examples/tokenize/tokenize.cpp b/examples/tokenize/tokenize.cpp index 8b1baea80..54c9834af 100644 --- a/examples/tokenize/tokenize.cpp +++ b/examples/tokenize/tokenize.cpp @@ -3,40 +3,390 @@ #include #include +#include #include #include -int main(int argc, char ** argv) { - if (argc < 3 || argv[1][0] == '-') { - printf("usage: %s MODEL_PATH PROMPT [--ids]\n" , argv[0]); +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#include // For CommandLineToArgvW +#endif + +static void print_usage_information(const char * argv0, FILE * stream) { + fprintf(stream, "usage: %s [options]\n\n", argv0); + fprintf(stream, "The tokenize program tokenizes a prompt using a given model,\n"); + fprintf(stream, "and prints the resulting tokens to standard output.\n\n"); + fprintf(stream, "It needs a model file, a prompt, and optionally other flags\n"); + fprintf(stream, "to control the behavior of the tokenizer.\n\n"); + fprintf(stream, " The possible options are:\n"); + fprintf(stream, "\n"); + fprintf(stream, " -h, --help print this help and exit\n"); + fprintf(stream, " -m MODEL_PATH, --model MODEL_PATH path to model.\n"); + fprintf(stream, " --ids if given, only print numerical token IDs, and not token strings.\n"); + fprintf(stream, " The output format looks like [1, 2, 3], i.e. parseable by Python.\n"); + fprintf(stream, " -f PROMPT_FNAME, --file PROMPT_FNAME read prompt from a file.\n"); + fprintf(stream, " -p PROMPT, --prompt PROMPT read prompt from the argument.\n"); + fprintf(stream, " --stdin read prompt from standard input.\n"); + fprintf(stream, " --no-bos do not ever add a BOS token to the prompt, even if normally the model uses a BOS token.\n"); + fprintf(stream, " --log-disable disable logs. Makes stderr quiet when loading the model.\n"); +} + +static void llama_log_callback_null(ggml_log_level level, const char * text, void * user_data) { + (void) level; + (void) text; + (void) user_data; +} + +static std::string read_prompt_from_file(const char * filepath, bool & success) { + success = false; + + std::ifstream in(filepath, std::ios::binary); + if (!in) { + fprintf(stderr, "%s: could not open file '%s' for reading: %s\n", __func__, filepath, strerror(errno)); + return std::string(); + } + // do not assume the file is seekable (e.g. /dev/stdin) + std::stringstream buffer; + buffer << in.rdbuf(); + if (in.fail()) { + fprintf(stderr, "%s: could not read the entire file '%s': %s\n", __func__, filepath, strerror(errno)); + return std::string(); + } + + success = true; + return buffer.str(); +} + +// +// Function: ingest_args(...) -> vector +// +// Takes argc and argv arguments, and converts them to a vector of UTF-8 encoded +// strings, as an STL vector. +// +// In particular, it handles character encoding shenanigans on Windows. +// +// Note: raw_argc and raw_argv are not actually read at all on Windows. +// On Windows we call GetCommandLineW to get the arguments in wchar_t +// format, ignoring the regular argc/argv arguments to main(). +// +// TODO: potential opportunity to roll common stuff into common/console.cpp +// in relation to Windows wchar_t shenanigans. +static std::vector ingest_args(int raw_argc, char ** raw_argv) { + std::vector argv; + + // Handle Windows, if given non-ASCII arguments. + // We convert wchar_t arguments into UTF-8 char* on this platform. + // Lets you invoke 'tokenize' on Windows cmd.exe with non-ASCII characters + // without throwing tantrums. +#if defined(_WIN32) + int argc; + const LPWSTR cmdline_wargv = GetCommandLineW(); + LPWSTR * wargv = CommandLineToArgvW(cmdline_wargv, &argc); + + // silence unused arg warnings + (void) raw_argc; + (void) raw_argv; + + for (int i = 0; i < argc; ++i) { + int length_needed = WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), 0, 0, NULL, NULL); + char * output_buf = (char *) calloc(length_needed+1, sizeof(char)); + GGML_ASSERT(output_buf); + + WideCharToMultiByte(CP_UTF8, 0, wargv[i], wcslen(wargv[i]), output_buf, length_needed, NULL, NULL); + output_buf[length_needed] = '\0'; + + argv.push_back(output_buf); + free(output_buf); + } + + LocalFree((HLOCAL) wargv); +#else + int argc = raw_argc; + for (int i = 0; i < argc; ++i) { + argv.push_back(raw_argv[i]); + } +#endif + + GGML_ASSERT((unsigned int) argc == argv.size()); + + return argv; +} + +// +// Function: write_utf8_cstr_to_stdout(const char *) -> +// +// writes a string to standard output; taking into account that on Windows +// to display correctly you have to use special handling. Works even if the +// user has not set a unicode code page on a Windows cmd.exe. +// +// In case of invalid UTF-8, invalid_utf8 is set to true on Windows, and something +// a human-readable is written instead. +// +// On non-Windows systems, simply printfs() the string. +static void write_utf8_cstr_to_stdout(const char * str, bool & invalid_utf8) { + invalid_utf8 = false; + +#if defined(_WIN32) + // Are we in a console? + HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE); + DWORD dwMode = 0; + + // According to Microsoft docs: + // "WriteConsole fails if it is used with a standard handle that is redirected to a file." + // Also according to the docs, you can use GetConsoleMode to check for that. + if (hConsole == INVALID_HANDLE_VALUE || !GetConsoleMode(hConsole, &dwMode)) { + printf("%s", str); + return; + } + + // MultiByteToWideChar reports an error if str is empty, don't report + // them as invalid_utf8. + if (*str == 0) { + return; + } + int length_needed = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, str, strlen(str), NULL, 0); + if (length_needed == 0) { + DWORD err = GetLastError(); + if (err == ERROR_NO_UNICODE_TRANSLATION) { + invalid_utf8 = true; + int len = strlen(str); + printf("<"); + for (int i = 0; i < len; ++i) { + if (i > 0) { + printf(" "); + } + printf("%02x", (uint8_t) str[i]); + } + printf(">"); + return; + } + GGML_ASSERT(false && "MultiByteToWideChar() failed in an unexpected way."); + } + + LPWSTR wstr = (LPWSTR) calloc(length_needed+1, sizeof(*wstr)); + GGML_ASSERT(wstr); + + MultiByteToWideChar(CP_UTF8, 0, str, strlen(str), wstr, length_needed); + WriteConsoleW(hConsole, wstr, length_needed, NULL, NULL); + + free(wstr); +#else + // TODO: reporting invalid_utf8 would be useful on non-Windows too. + // printf will silently just write bad unicode. + printf("%s", str); +#endif +} + +int main(int raw_argc, char ** raw_argv) { + const std::vector argv = ingest_args(raw_argc, raw_argv); + const int argc = argv.size(); + + if (argc <= 1) { + print_usage_information(argv[0].c_str(), stderr); return 1; } - const char * model_path = argv[1]; - const char * prompt = argv[2]; + ////// + // Read out all the command line arguments. + ////// - const bool printing_ids = argc > 3 && std::string(argv[3]) == "--ids"; + // variables where to put any arguments we see. + bool printing_ids = false; + bool no_bos = false; + bool disable_logging = false; + const char * model_path = NULL; + const char * prompt_path = NULL; + const char * prompt_arg = NULL; + + // track which arguments were explicitly given + // used for sanity checking down the line + bool model_path_set = false; + bool prompt_path_set = false; + bool prompt_set = false; + bool stdin_set = false; + + int iarg = 1; + for (; iarg < argc; ++iarg) { + std::string arg{argv[iarg]}; + if (arg == "-h" || arg == "--help") { + print_usage_information(argv[0].c_str(), stdout); + return 0; + } + else if (arg == "--ids") { + printing_ids = true; + } + else if (arg == "-m" || arg == "--model") { + if (model_path_set) { + fprintf(stderr, "Error: -m or --model specified multiple times.\n"); + return 1; + } + model_path = argv[++iarg].c_str(); + model_path_set = true; + } + else if (arg == "--no-bos") { + no_bos = true; + } + else if (arg == "-p" || arg == "--prompt") { + if (prompt_set) { + fprintf(stderr, "Error: -p or --prompt specified multiple times.\n"); + return 1; + } + prompt_arg = argv[++iarg].c_str(); + prompt_set = true; + } + else if (arg == "-f" || arg == "--file") { + if (prompt_path_set) { + fprintf(stderr, "Error: -f or --file specified multiple times.\n"); + return 1; + } + prompt_path = argv[++iarg].c_str(); + prompt_path_set = true; + } + else if (arg == "--stdin") { + stdin_set = true; + } + else if (arg == "--log-disable") { + disable_logging = true; + } + else { + fprintf(stderr, "Error: unknown option '%s'\n", argv[iarg].c_str()); + return 1; + } + } + + ////// + // Sanity check the command line arguments. + ////// + + // Check that we have the required stuff set. + if (model_path_set && model_path == NULL) { + fprintf(stderr, "Error: --model requires an argument.\n"); + return 1; + } + if (!model_path_set) { + fprintf(stderr, "Error: must specify --model.\n"); + return 1; + } + if (prompt_path_set && prompt_path == NULL) { + fprintf(stderr, "Error: --file requires an argument.\n"); + return 1; + } + if (prompt_set && prompt_arg == NULL) { + fprintf(stderr, "Error: --prompt requires an argument.\n"); + return 1; + } + const int prompts_set = !!(prompt_path_set) + !!(prompt_set) + !!(stdin_set); + if (prompts_set > 1) { + fprintf(stderr, "Error: --stdin, --file and --prompt are mutually exclusive.\n"); + return 1; + } + // Must have some prompt. + if (prompts_set == 0) { + fprintf(stderr, "Error: must specify one of: --stdin, --file or --prompt.\n"); + return 1; + } + + GGML_ASSERT(model_path); + GGML_ASSERT(prompt_path || prompt_arg || stdin_set); + + ////// + // Figure out where will the prompt come from. + ////// + + std::string prompt; + if (prompt_path_set) { + bool success = false; + prompt = read_prompt_from_file(prompt_path, success); + if (!success) { + return 1; + } + } else if (prompt_set) { + prompt = prompt_arg; + } else { + GGML_ASSERT(stdin_set); + // we read stdin *after* loading model (early exit if model cannot + // be loaded, which can be a nicer user experience) + } + + ////// + // Start actually doing the tokenizing stuff. + ////// + +#ifdef LOG_DISABLE_LOGS + disable_logging = true; +#endif + + if (disable_logging) { + llama_log_set(llama_log_callback_null, NULL); + } llama_backend_init(); llama_model_params model_params = llama_model_default_params(); model_params.vocab_only = true; llama_model * model = llama_load_model_from_file(model_path, model_params); + if (!model) { + fprintf(stderr, "Error: could not load model from file '%s'.\n", model_path); + return 1; + } llama_context_params ctx_params = llama_context_default_params(); llama_context * ctx = llama_new_context_with_model(model, ctx_params); + if (!ctx) { + fprintf(stderr, "Error: could not create context.\n"); + return 1; + } + + // read entire prompt from stdin? + if (stdin_set) { + GGML_ASSERT(!prompt_path_set && !prompt_set); + + std::stringstream stdin_buffer; + stdin_buffer << std::cin.rdbuf(); + if (std::cin.fail()) { + fprintf(stderr, "Error: could not read the entire standard input.\n"); + return 1; + } + + prompt = stdin_buffer.str(); + } + + const bool model_wants_add_bos = llama_should_add_bos_token(model); + const bool add_bos = model_wants_add_bos && !no_bos; std::vector tokens; + tokens = ::llama_tokenize(model, prompt, add_bos, true); - tokens = ::llama_tokenize(model, prompt, true, true); + if (printing_ids) { + printf("["); + } for (int i = 0; i < (int) tokens.size(); i++) { if (printing_ids) { - printf("%d\n", tokens[i]); + if (i > 0) { + printf(", "); + } + printf("%d", tokens[i]); } else { - printf("%6d -> '%s'\n", tokens[i], llama_token_to_piece(ctx, tokens[i]).c_str()); + bool invalid_utf8 = false; + printf("%6d -> '", tokens[i]); + write_utf8_cstr_to_stdout(llama_token_to_piece(ctx, tokens[i]).c_str(), invalid_utf8); + if (invalid_utf8) { + printf("' (utf-8 decode failure)\n"); + } else { + printf("'\n"); + } } } + if (printing_ids) { + printf("]\n"); + } + + // silence valgrind + llama_free(ctx); + llama_free_model(model); + return 0; } From 902184dd3a9d6685e752b19027a48423742531db Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Sat, 25 May 2024 05:30:59 +0200 Subject: [PATCH 132/137] fix missing slash in `fs_get_cache_directory()` (#7503) * fix missing slash in fs_get_cache_directory() * use LOCALAPPDATA for fs_get_cache_directory() * better code style --- common/common.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 7500e08ff..401d72bac 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1855,11 +1855,15 @@ bool fs_create_directory_with_parents(const std::string & path) { std::string fs_get_cache_directory() { std::string cache_directory = ""; + auto ensure_trailing_slash = [](std::string p) { + // Make sure to add trailing slash + if (p.back() != DIRECTORY_SEPARATOR) { + p += DIRECTORY_SEPARATOR; + } + return p; + }; if (getenv("LLAMA_CACHE")) { cache_directory = std::getenv("LLAMA_CACHE"); - if (cache_directory.back() != DIRECTORY_SEPARATOR) { - cache_directory += DIRECTORY_SEPARATOR; - } } else { #ifdef __linux__ if (std::getenv("XDG_CACHE_HOME")) { @@ -1870,12 +1874,12 @@ std::string fs_get_cache_directory() { #elif defined(__APPLE__) cache_directory = std::getenv("HOME") + std::string("/Library/Caches/"); #elif defined(_WIN32) - cache_directory = std::getenv("APPDATA"); + cache_directory = std::getenv("LOCALAPPDATA"); #endif // __linux__ + cache_directory = ensure_trailing_slash(cache_directory); cache_directory += "llama.cpp"; - cache_directory += DIRECTORY_SEPARATOR; } - return cache_directory; + return ensure_trailing_slash(cache_directory); } From 9791f402580838d7f8543ae7bc633ef265e436f0 Mon Sep 17 00:00:00 2001 From: Elton Kola Date: Sat, 25 May 2024 04:11:33 -0400 Subject: [PATCH 133/137] android : module (#7502) * move ndk code to a new library * add gradle file --- examples/llama.android/app/build.gradle.kts | 25 +------ .../java/com/example/llama/MainViewModel.kt | 13 ++-- examples/llama.android/build.gradle.kts | 1 + examples/llama.android/llama/.gitignore | 1 + .../src/main/cpp => llama}/CMakeLists.txt | 2 +- examples/llama.android/llama/build.gradle.kts | 68 +++++++++++++++++++ .../llama.android/llama/consumer-rules.pro | 0 .../llama.android/llama/proguard-rules.pro | 21 ++++++ .../llama/cpp/ExampleInstrumentedTest.kt | 24 +++++++ .../llama/src/main/AndroidManifest.xml | 4 ++ .../llama/src/main/cpp/CMakeLists.txt | 49 +++++++++++++ .../src/main/cpp/llama-android.cpp | 28 ++++---- .../java/android/llama/cpp/LLamaAndroid.kt} | 8 +-- .../java/android/llama/cpp/ExampleUnitTest.kt | 17 +++++ examples/llama.android/settings.gradle.kts | 1 + 15 files changed, 213 insertions(+), 49 deletions(-) create mode 100644 examples/llama.android/llama/.gitignore rename examples/llama.android/{app/src/main/cpp => llama}/CMakeLists.txt (98%) create mode 100644 examples/llama.android/llama/build.gradle.kts create mode 100644 examples/llama.android/llama/consumer-rules.pro create mode 100644 examples/llama.android/llama/proguard-rules.pro create mode 100644 examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt create mode 100644 examples/llama.android/llama/src/main/AndroidManifest.xml create mode 100644 examples/llama.android/llama/src/main/cpp/CMakeLists.txt rename examples/llama.android/{app => llama}/src/main/cpp/llama-android.cpp (92%) rename examples/llama.android/{app/src/main/java/com/example/llama/Llm.kt => llama/src/main/java/android/llama/cpp/LLamaAndroid.kt} (97%) create mode 100644 examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt diff --git a/examples/llama.android/app/build.gradle.kts b/examples/llama.android/app/build.gradle.kts index d42140efe..8d1b37195 100644 --- a/examples/llama.android/app/build.gradle.kts +++ b/examples/llama.android/app/build.gradle.kts @@ -7,8 +7,6 @@ android { namespace = "com.example.llama" compileSdk = 34 - ndkVersion = "26.1.10909125" - defaultConfig { applicationId = "com.example.llama" minSdk = 33 @@ -20,17 +18,6 @@ android { vectorDrawables { useSupportLibrary = true } - ndk { - // Add NDK properties if wanted, e.g. - // abiFilters += listOf("arm64-v8a") - } - externalNativeBuild { - cmake { - arguments += "-DCMAKE_BUILD_TYPE=Release" - cppFlags += listOf() - arguments += listOf() - } - } } buildTypes { @@ -55,17 +42,6 @@ android { composeOptions { kotlinCompilerExtensionVersion = "1.5.1" } - packaging { - resources { - excludes += "/META-INF/{AL2.0,LGPL2.1}" - } - } - externalNativeBuild { - cmake { - path = file("src/main/cpp/CMakeLists.txt") - version = "3.22.1" - } - } } dependencies { @@ -78,6 +54,7 @@ dependencies { implementation("androidx.compose.ui:ui-graphics") implementation("androidx.compose.ui:ui-tooling-preview") implementation("androidx.compose.material3:material3") + implementation(project(":llama")) testImplementation("junit:junit:4.13.2") androidTestImplementation("androidx.test.ext:junit:1.1.5") androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt index be95e2221..45ac29938 100644 --- a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt +++ b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt @@ -1,5 +1,6 @@ package com.example.llama +import android.llama.cpp.LLamaAndroid import android.util.Log import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -9,7 +10,7 @@ import androidx.lifecycle.viewModelScope import kotlinx.coroutines.flow.catch import kotlinx.coroutines.launch -class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { +class MainViewModel(private val llamaAndroid: LLamaAndroid = LLamaAndroid.instance()): ViewModel() { companion object { @JvmStatic private val NanosPerSecond = 1_000_000_000.0 @@ -28,7 +29,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { viewModelScope.launch { try { - llm.unload() + llamaAndroid.unload() } catch (exc: IllegalStateException) { messages += exc.message!! } @@ -44,7 +45,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { messages += "" viewModelScope.launch { - llm.send(text) + llamaAndroid.send(text) .catch { Log.e(tag, "send() failed", it) messages += it.message!! @@ -57,7 +58,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { viewModelScope.launch { try { val start = System.nanoTime() - val warmupResult = llm.bench(pp, tg, pl, nr) + val warmupResult = llamaAndroid.bench(pp, tg, pl, nr) val end = System.nanoTime() messages += warmupResult @@ -70,7 +71,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { return@launch } - messages += llm.bench(512, 128, 1, 3) + messages += llamaAndroid.bench(512, 128, 1, 3) } catch (exc: IllegalStateException) { Log.e(tag, "bench() failed", exc) messages += exc.message!! @@ -81,7 +82,7 @@ class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() { fun load(pathToModel: String) { viewModelScope.launch { try { - llm.load(pathToModel) + llamaAndroid.load(pathToModel) messages += "Loaded $pathToModel" } catch (exc: IllegalStateException) { Log.e(tag, "load() failed", exc) diff --git a/examples/llama.android/build.gradle.kts b/examples/llama.android/build.gradle.kts index 50ebc8211..acd1ada7d 100644 --- a/examples/llama.android/build.gradle.kts +++ b/examples/llama.android/build.gradle.kts @@ -2,4 +2,5 @@ plugins { id("com.android.application") version "8.2.0" apply false id("org.jetbrains.kotlin.android") version "1.9.0" apply false + id("com.android.library") version "8.2.0" apply false } diff --git a/examples/llama.android/llama/.gitignore b/examples/llama.android/llama/.gitignore new file mode 100644 index 000000000..796b96d1c --- /dev/null +++ b/examples/llama.android/llama/.gitignore @@ -0,0 +1 @@ +/build diff --git a/examples/llama.android/app/src/main/cpp/CMakeLists.txt b/examples/llama.android/llama/CMakeLists.txt similarity index 98% rename from examples/llama.android/app/src/main/cpp/CMakeLists.txt rename to examples/llama.android/llama/CMakeLists.txt index 4536974a5..a5618cac0 100644 --- a/examples/llama.android/app/src/main/cpp/CMakeLists.txt +++ b/examples/llama.android/llama/CMakeLists.txt @@ -42,7 +42,7 @@ add_subdirectory(../../../../../../ build-llama) # used in the AndroidManifest.xml file. add_library(${CMAKE_PROJECT_NAME} SHARED # List C/C++ source files with relative paths to this CMakeLists.txt. - llama-android.cpp) + llama-android.cpp) # Specifies libraries CMake should link to your target library. You # can link libraries from various origins, such as libraries defined in this diff --git a/examples/llama.android/llama/build.gradle.kts b/examples/llama.android/llama/build.gradle.kts new file mode 100644 index 000000000..0a3806172 --- /dev/null +++ b/examples/llama.android/llama/build.gradle.kts @@ -0,0 +1,68 @@ +plugins { + id("com.android.library") + id("org.jetbrains.kotlin.android") +} + +android { + namespace = "android.llama.cpp" + compileSdk = 34 + + defaultConfig { + minSdk = 33 + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + consumerProguardFiles("consumer-rules.pro") + ndk { + // Add NDK properties if wanted, e.g. + // abiFilters += listOf("arm64-v8a") + } + externalNativeBuild { + cmake { + arguments += "-DCMAKE_BUILD_TYPE=Release" + cppFlags += listOf() + arguments += listOf() + + cppFlags("") + } + } + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + externalNativeBuild { + cmake { + path("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } + } + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } + + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + } + } +} + +dependencies { + + implementation("androidx.core:core-ktx:1.12.0") + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.11.0") + testImplementation("junit:junit:4.13.2") + androidTestImplementation("androidx.test.ext:junit:1.1.5") + androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1") +} diff --git a/examples/llama.android/llama/consumer-rules.pro b/examples/llama.android/llama/consumer-rules.pro new file mode 100644 index 000000000..e69de29bb diff --git a/examples/llama.android/llama/proguard-rules.pro b/examples/llama.android/llama/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/examples/llama.android/llama/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt b/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt new file mode 100644 index 000000000..05d6ab5d2 --- /dev/null +++ b/examples/llama.android/llama/src/androidTest/java/android/llama/cpp/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package android.llama.cpp + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("android.llama.cpp.test", appContext.packageName) + } +} diff --git a/examples/llama.android/llama/src/main/AndroidManifest.xml b/examples/llama.android/llama/src/main/AndroidManifest.xml new file mode 100644 index 000000000..8bdb7e14b --- /dev/null +++ b/examples/llama.android/llama/src/main/AndroidManifest.xml @@ -0,0 +1,4 @@ + + + + diff --git a/examples/llama.android/llama/src/main/cpp/CMakeLists.txt b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..42ebaad49 --- /dev/null +++ b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt @@ -0,0 +1,49 @@ +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html. +# For more examples on how to use CMake, see https://github.com/android/ndk-samples. + +# Sets the minimum CMake version required for this project. +cmake_minimum_required(VERSION 3.22.1) + +# Declares the project name. The project name can be accessed via ${ PROJECT_NAME}, +# Since this is the top level CMakeLists.txt, the project name is also accessible +# with ${CMAKE_PROJECT_NAME} (both CMake variables are in-sync within the top level +# build script scope). +project("llama-android") + +include(FetchContent) +FetchContent_Declare( + llama + GIT_REPOSITORY https://github.com/ggerganov/llama.cpp + GIT_TAG master +) + +# Also provides "common" +FetchContent_MakeAvailable(llama) + +# Creates and names a library, sets it as either STATIC +# or SHARED, and provides the relative paths to its source code. +# You can define multiple libraries, and CMake builds them for you. +# Gradle automatically packages shared libraries with your APK. +# +# In this top level CMakeLists.txt, ${CMAKE_PROJECT_NAME} is used to define +# the target library name; in the sub-module's CMakeLists.txt, ${PROJECT_NAME} +# is preferred for the same purpose. +# +# In order to load a library into your app from Java/Kotlin, you must call +# System.loadLibrary() and pass the name of the library defined here; +# for GameActivity/NativeActivity derived applications, the same library name must be +# used in the AndroidManifest.xml file. +add_library(${CMAKE_PROJECT_NAME} SHARED + # List C/C++ source files with relative paths to this CMakeLists.txt. + llama-android.cpp) + +# Specifies libraries CMake should link to your target library. You +# can link libraries from various origins, such as libraries defined in this +# build script, prebuilt third-party libraries, or Android system libraries. +target_link_libraries(${CMAKE_PROJECT_NAME} + # List libraries link to the target library + llama + common + android + log) diff --git a/examples/llama.android/app/src/main/cpp/llama-android.cpp b/examples/llama.android/llama/src/main/cpp/llama-android.cpp similarity index 92% rename from examples/llama.android/app/src/main/cpp/llama-android.cpp rename to examples/llama.android/llama/src/main/cpp/llama-android.cpp index 4af9de303..874158ef0 100644 --- a/examples/llama.android/app/src/main/cpp/llama-android.cpp +++ b/examples/llama.android/llama/src/main/cpp/llama-android.cpp @@ -81,7 +81,7 @@ static void log_callback(ggml_log_level level, const char * fmt, void * data) { extern "C" JNIEXPORT jlong JNICALL -Java_com_example_llama_Llm_load_1model(JNIEnv *env, jobject, jstring filename) { +Java_android_llama_cpp_LLamaAndroid_load_1model(JNIEnv *env, jobject, jstring filename) { llama_model_params model_params = llama_model_default_params(); auto path_to_model = env->GetStringUTFChars(filename, 0); @@ -101,13 +101,13 @@ Java_com_example_llama_Llm_load_1model(JNIEnv *env, jobject, jstring filename) { extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_free_1model(JNIEnv *, jobject, jlong model) { +Java_android_llama_cpp_LLamaAndroid_free_1model(JNIEnv *, jobject, jlong model) { llama_free_model(reinterpret_cast(model)); } extern "C" JNIEXPORT jlong JNICALL -Java_com_example_llama_Llm_new_1context(JNIEnv *env, jobject, jlong jmodel) { +Java_android_llama_cpp_LLamaAndroid_new_1context(JNIEnv *env, jobject, jlong jmodel) { auto model = reinterpret_cast(jmodel); if (!model) { @@ -139,25 +139,25 @@ Java_com_example_llama_Llm_new_1context(JNIEnv *env, jobject, jlong jmodel) { extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_free_1context(JNIEnv *, jobject, jlong context) { +Java_android_llama_cpp_LLamaAndroid_free_1context(JNIEnv *, jobject, jlong context) { llama_free(reinterpret_cast(context)); } extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_backend_1free(JNIEnv *, jobject) { +Java_android_llama_cpp_LLamaAndroid_backend_1free(JNIEnv *, jobject) { llama_backend_free(); } extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_log_1to_1android(JNIEnv *, jobject) { +Java_android_llama_cpp_LLamaAndroid_log_1to_1android(JNIEnv *, jobject) { llama_log_set(log_callback, NULL); } extern "C" JNIEXPORT jstring JNICALL -Java_com_example_llama_Llm_bench_1model( +Java_android_llama_cpp_LLamaAndroid_bench_1model( JNIEnv *env, jobject, jlong context_pointer, @@ -271,13 +271,13 @@ Java_com_example_llama_Llm_bench_1model( extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_free_1batch(JNIEnv *, jobject, jlong batch_pointer) { +Java_android_llama_cpp_LLamaAndroid_free_1batch(JNIEnv *, jobject, jlong batch_pointer) { llama_batch_free(*reinterpret_cast(batch_pointer)); } extern "C" JNIEXPORT jlong JNICALL -Java_com_example_llama_Llm_new_1batch(JNIEnv *, jobject, jint n_tokens, jint embd, jint n_seq_max) { +Java_android_llama_cpp_LLamaAndroid_new_1batch(JNIEnv *, jobject, jint n_tokens, jint embd, jint n_seq_max) { // Source: Copy of llama.cpp:llama_batch_init but heap-allocated. @@ -313,19 +313,19 @@ Java_com_example_llama_Llm_new_1batch(JNIEnv *, jobject, jint n_tokens, jint emb extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_backend_1init(JNIEnv *, jobject) { +Java_android_llama_cpp_LLamaAndroid_backend_1init(JNIEnv *, jobject) { llama_backend_init(); } extern "C" JNIEXPORT jstring JNICALL -Java_com_example_llama_Llm_system_1info(JNIEnv *env, jobject) { +Java_android_llama_cpp_LLamaAndroid_system_1info(JNIEnv *env, jobject) { return env->NewStringUTF(llama_print_system_info()); } extern "C" JNIEXPORT jint JNICALL -Java_com_example_llama_Llm_completion_1init( +Java_android_llama_cpp_LLamaAndroid_completion_1init( JNIEnv *env, jobject, jlong context_pointer, @@ -376,7 +376,7 @@ Java_com_example_llama_Llm_completion_1init( extern "C" JNIEXPORT jstring JNICALL -Java_com_example_llama_Llm_completion_1loop( +Java_android_llama_cpp_LLamaAndroid_completion_1loop( JNIEnv * env, jobject, jlong context_pointer, @@ -438,6 +438,6 @@ Java_com_example_llama_Llm_completion_1loop( extern "C" JNIEXPORT void JNICALL -Java_com_example_llama_Llm_kv_1cache_1clear(JNIEnv *, jobject, jlong context) { +Java_android_llama_cpp_LLamaAndroid_kv_1cache_1clear(JNIEnv *, jobject, jlong context) { llama_kv_cache_clear(reinterpret_cast(context)); } diff --git a/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt b/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt similarity index 97% rename from examples/llama.android/app/src/main/java/com/example/llama/Llm.kt rename to examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt index d86afee37..6c63e54e0 100644 --- a/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt +++ b/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt @@ -1,4 +1,4 @@ -package com.example.llama +package android.llama.cpp import android.util.Log import kotlinx.coroutines.CoroutineDispatcher @@ -10,7 +10,7 @@ import kotlinx.coroutines.withContext import java.util.concurrent.Executors import kotlin.concurrent.thread -class Llm { +class LLamaAndroid { private val tag: String? = this::class.simpleName private val threadLocalState: ThreadLocal = ThreadLocal.withInitial { State.Idle } @@ -165,8 +165,8 @@ class Llm { } // Enforce only one instance of Llm. - private val _instance: Llm = Llm() + private val _instance: LLamaAndroid = LLamaAndroid() - fun instance(): Llm = _instance + fun instance(): LLamaAndroid = _instance } } diff --git a/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt b/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt new file mode 100644 index 000000000..cbbb974d3 --- /dev/null +++ b/examples/llama.android/llama/src/test/java/android/llama/cpp/ExampleUnitTest.kt @@ -0,0 +1,17 @@ +package android.llama.cpp + +import org.junit.Test + +import org.junit.Assert.* + +/** + * Example local unit test, which will execute on the development machine (host). + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +class ExampleUnitTest { + @Test + fun addition_isCorrect() { + assertEquals(4, 2 + 2) + } +} diff --git a/examples/llama.android/settings.gradle.kts b/examples/llama.android/settings.gradle.kts index 2ba32c4fa..c7c1a034a 100644 --- a/examples/llama.android/settings.gradle.kts +++ b/examples/llama.android/settings.gradle.kts @@ -15,3 +15,4 @@ dependencyResolutionManagement { rootProject.name = "LlamaAndroid" include(":app") +include(":llama") From faa0e6979a11dcb731e9d778ad42ceaa0302015e Mon Sep 17 00:00:00 2001 From: "Masaya, Kato" <62578291+msy-kato@users.noreply.github.com> Date: Sat, 25 May 2024 17:42:31 +0900 Subject: [PATCH 134/137] ggml: aarch64: SVE kernels for q8_0_q8_0, q4_0_q8_0 vector dot (#7433) * Add SVE support for q4_0_q8_0 q8_0_q8_0 * remove ifdef --- CMakeLists.txt | 4 +++ common/common.cpp | 1 + ggml-impl.h | 4 +++ ggml-quants.c | 66 +++++++++++++++++++++++++++++++++++++++++++++-- ggml.c | 10 +++++++ ggml.h | 1 + llama.cpp | 1 + 7 files changed, 85 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ef02ff669..c5add8239 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ else() set(INS_ENB ON) endif() +option(LLAMA_SVE "llama: enable SVE" OFF) option(LLAMA_AVX "llama: enable AVX" ${INS_ENB}) option(LLAMA_AVX2 "llama: enable AVX2" ${INS_ENB}) option(LLAMA_AVX512 "llama: enable AVX512" OFF) @@ -1040,6 +1041,9 @@ if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STR # Raspberry Pi 3, 4, Zero 2 (32-bit) list(APPEND ARCH_FLAGS -mno-unaligned-access) endif() + if (LLAMA_SVE) + list(APPEND ARCH_FLAGS -march=armv8.6-a+sve) + endif() endif() elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND diff --git a/common/common.cpp b/common/common.cpp index 401d72bac..c64590385 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -2844,6 +2844,7 @@ void yaml_dump_non_result_info(FILE * stream, const gpt_params & params, const l fprintf(stream, "cpu_has_fma: %s\n", ggml_cpu_has_fma() ? "true" : "false"); fprintf(stream, "cpu_has_gpublas: %s\n", ggml_cpu_has_gpublas() ? "true" : "false"); fprintf(stream, "cpu_has_neon: %s\n", ggml_cpu_has_neon() ? "true" : "false"); + fprintf(stream, "cpu_has_sve: %s\n", ggml_cpu_has_sve() ? "true" : "false"); fprintf(stream, "cpu_has_f16c: %s\n", ggml_cpu_has_f16c() ? "true" : "false"); fprintf(stream, "cpu_has_fp16_va: %s\n", ggml_cpu_has_fp16_va() ? "true" : "false"); fprintf(stream, "cpu_has_wasm_simd: %s\n", ggml_cpu_has_wasm_simd() ? "true" : "false"); diff --git a/ggml-impl.h b/ggml-impl.h index 362d40f4d..5e77471f3 100644 --- a/ggml-impl.h +++ b/ggml-impl.h @@ -144,6 +144,10 @@ extern "C" { #endif #endif +#if defined(__ARM_FEATURE_SVE) +#include +#endif + // 16-bit float // on Arm, we use __fp16 // on x86, we use uint16_t diff --git a/ggml-quants.c b/ggml-quants.c index bb01ce93c..4f2c7224c 100644 --- a/ggml-quants.c +++ b/ggml-quants.c @@ -3813,7 +3813,44 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r return; } #endif -#if defined(__ARM_NEON) +#if defined(__ARM_FEATURE_SVE) + const svbool_t ptrueh = svptrue_pat_b8(SV_VL16); + const svbool_t ptruel = svnot_b_z(svptrue_b8(), ptrueh); + + svfloat32_t sumv0 = svdup_n_f32(0.0f); + svfloat32_t sumv1 = svdup_n_f32(0.0f); + + assert(nb % 2 == 0); // TODO: handle odd nb + + for (int i = 0; i < nb; i += 2) { + const block_q4_0 * restrict x0 = &x[i + 0]; + const block_q4_0 * restrict x1 = &x[i + 1]; + const block_q8_0 * restrict y0 = &y[i + 0]; + const block_q8_0 * restrict y1 = &y[i + 1]; + + // load x + const svuint8_t qx0r = svld1rq_u8(svptrue_b8(), x0->qs); + const svuint8_t qx1r = svld1rq_u8(svptrue_b8(), x1->qs); + + // 4-bit -> 8-bit + const svint8_t qx0 = svreinterpret_s8_u8(svlsr_n_u8_m(ptruel, svand_n_u8_m(ptrueh, qx0r, 0x0F), 0x04)); + const svint8_t qx1 = svreinterpret_s8_u8(svlsr_n_u8_m(ptruel, svand_n_u8_m(ptrueh, qx1r, 0x0F), 0x04)); + + // sub 8 + const svint8_t qx0s = svsub_n_s8_x(svptrue_b8(), qx0, 8); + const svint8_t qx1s = svsub_n_s8_x(svptrue_b8(), qx1, 8); + + // load y + const svint8_t qy0 = svld1_s8(svptrue_b8(), y0->qs); + const svint8_t qy1 = svld1_s8(svptrue_b8(), y1->qs); + + // dot product + sumv0 = svmla_n_f32_x(svptrue_b32(), sumv0, svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx0s, qy0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d)); + sumv1 = svmla_n_f32_x(svptrue_b32(), sumv1, svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx1s, qy1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d)); + } + + *s = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1)); +#elif defined(__ARM_NEON) float32x4_t sumv0 = vdupq_n_f32(0.0f); float32x4_t sumv1 = vdupq_n_f32(0.0f); @@ -5384,7 +5421,32 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r return; } #endif -#if defined(__ARM_NEON) +#if defined(__ARM_FEATURE_SVE) + svfloat32_t sumv0 = svdup_n_f32(0.0f); + svfloat32_t sumv1 = svdup_n_f32(0.0f); + + assert(nb % 2 == 0); // TODO: handle odd nb + + for (int i = 0; i < nb; i += 2) { + const block_q8_0 * restrict x0 = &x[i + 0]; + const block_q8_0 * restrict x1 = &x[i + 1]; + const block_q8_0 * restrict y0 = &y[i + 0]; + const block_q8_0 * restrict y1 = &y[i + 1]; + + // load x + const svint8_t qx0 = svld1_s8(svptrue_b8(), x0->qs); + const svint8_t qx1 = svld1_s8(svptrue_b8(), x1->qs); + + // load y + const svint8_t qy0 = svld1_s8(svptrue_b8(), y0->qs); + const svint8_t qy1 = svld1_s8(svptrue_b8(), y1->qs); + + sumv0 = svmla_n_f32_x(svptrue_b32(), sumv0, svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx0, qy0)), GGML_FP16_TO_FP32(x0->d)*GGML_FP16_TO_FP32(y0->d)); + sumv1 = svmla_n_f32_x(svptrue_b32(), sumv1, svcvt_f32_s32_x(svptrue_b32(), svdot_s32(svdup_n_s32(0), qx1, qy1)), GGML_FP16_TO_FP32(x1->d)*GGML_FP16_TO_FP32(y1->d)); + } + + *s = svaddv_f32(svptrue_b32(), svadd_f32_x(svptrue_b32(), sumv0, sumv1)); +#elif defined(__ARM_NEON) float32x4_t sumv0 = vdupq_n_f32(0.0f); float32x4_t sumv1 = vdupq_n_f32(0.0f); diff --git a/ggml.c b/ggml.c index 9e72b7a76..5145ceec9 100644 --- a/ggml.c +++ b/ggml.c @@ -22742,6 +22742,16 @@ int ggml_cpu_has_neon(void) { #endif } +int ggml_cpu_has_sve(void) { +#if defined(__ARM_FEATURE_SVE) + // TODO: Currently, SVE 256 bit is only supported. + GGML_ASSERT(svcntb() == QK8_0); + return 1; +#else + return 0; +#endif +} + int ggml_cpu_has_arm_fma(void) { #if defined(__ARM_FEATURE_FMA) return 1; diff --git a/ggml.h b/ggml.h index be81e0c52..f803ba724 100644 --- a/ggml.h +++ b/ggml.h @@ -2404,6 +2404,7 @@ extern "C" { GGML_API int ggml_cpu_has_avx512_bf16(void); GGML_API int ggml_cpu_has_fma (void); GGML_API int ggml_cpu_has_neon (void); + GGML_API int ggml_cpu_has_sve (void); GGML_API int ggml_cpu_has_arm_fma (void); GGML_API int ggml_cpu_has_metal (void); GGML_API int ggml_cpu_has_f16c (void); diff --git a/llama.cpp b/llama.cpp index 3c9fe15bb..85cb3140d 100644 --- a/llama.cpp +++ b/llama.cpp @@ -18337,6 +18337,7 @@ const char * llama_print_system_info(void) { s += "AVX512_BF16 = " + std::to_string(ggml_cpu_has_avx512_bf16()) + " | "; s += "FMA = " + std::to_string(ggml_cpu_has_fma()) + " | "; s += "NEON = " + std::to_string(ggml_cpu_has_neon()) + " | "; + s += "SVE = " + std::to_string(ggml_cpu_has_sve()) + " | "; s += "ARM_FMA = " + std::to_string(ggml_cpu_has_arm_fma()) + " | "; s += "F16C = " + std::to_string(ggml_cpu_has_f16c()) + " | "; s += "FP16_VA = " + std::to_string(ggml_cpu_has_fp16_va()) + " | "; From 00c63907931bb08a0ed2b7e38cf44dd290143cb9 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Sat, 25 May 2024 05:04:03 -0400 Subject: [PATCH 135/137] main : don't print special tokens with --grammar (#6923) * main : don't print special tokens with --grammar The CLI interface was recently changed to print special control tokens like the stop message one. This token shouldn't be printed if the grammar flag was passed, unless the grammar specifies it, because that breaks shell-scriptability. * main: use seperate stream for control characters * main: use dprintf and add --ctrl-token-no-out and --ctrl-token-fd-out * main: dprintf isn't part of the IEEE POSIX standard. Just use write(). * main: remove --ctrl-token-fd-out in favor for fcntl() based detection * common.cpp: accidentally removed --interactive-first * main: only merge stdout and control token if not in conversation or grammar mode * main: rejig control token descriptor handling * main: must check pipe status on very top of program * main: renamed --no-special from --ctrl-token-no-out and other refactoring * main: refactor ctrl_token_no_out --> no_special * llama: rename llama_token_is_control_token() to llama_token_is_control() * main: remove special token file descriptor feature (#5) --------- Co-authored-by: Brian --- common/common.cpp | 5 +++++ common/common.h | 1 + examples/main/main.cpp | 20 +++++++++++++++++--- llama.cpp | 4 ++++ llama.h | 3 +++ 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index c64590385..781f2166b 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -904,6 +904,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa params.interactive_specials = true; return true; } + if (arg == "--no-special") { + params.no_special = true; + return true; + } if (arg == "--embedding") { params.embedding = true; return true; @@ -1364,6 +1368,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param printf(" -i, --interactive run in interactive mode\n"); printf(" --interactive-specials allow special tokens in user text, in interactive mode\n"); printf(" --interactive-first run in interactive mode and wait for input right away\n"); + printf(" --no-special control tokens output disabled\n"); printf(" -cnv, --conversation run in conversation mode (does not print special tokens and suffix/prefix)\n"); printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n"); printf(" -cml, --chatml run in chatml mode (use with ChatML-compatible models)\n"); diff --git a/common/common.h b/common/common.h index f68f3c297..5388f6b68 100644 --- a/common/common.h +++ b/common/common.h @@ -146,6 +146,7 @@ struct gpt_params { bool use_color = false; // use color to distinguish generations and inputs bool interactive = false; // interactive mode bool interactive_specials = false; // whether to allow special tokens from user, during interactive mode + bool no_special = false; // disable control token output bool conversation = false; // conversation mode (does not print special tokens and suffix/prefix) bool chatml = false; // chatml mode (used for models trained on chatml syntax) bool prompt_cache_all = false; // save user input and generations to prompt cache diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 09fa85fce..ac35772f1 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -740,18 +740,32 @@ int main(int argc, char ** argv) { // display text if (input_echo && display) { for (auto id : embd) { - const std::string token_str = llama_token_to_piece(ctx, id, !params.conversation); - printf("%s", token_str.c_str()); + const std::string token_str = llama_token_to_piece(ctx, id); + // Console/Stream Output + if (!llama_token_is_control(llama_get_model(ctx), id)) { + // Stream Output Token To Standard Output + fprintf(stdout, "%s", token_str.c_str()); + } else if (!params.no_special && !params.conversation) { + // Stream Control Token To Standard Output Stream + fprintf(stdout, "%s", token_str.c_str()); + } + + // Record Displayed Tokens To Log + // Note: Generated tokens are created one by one hence this check if (embd.size() > 1) { + // Incoming Requested Tokens input_tokens.push_back(id); } else { + // Outgoing Generated Tokens output_tokens.push_back(id); output_ss << token_str; } + + fflush(stdout); } - fflush(stdout); } + // reset color to default if there is no pending user input if (input_echo && (int) embd_inp.size() == n_consumed) { console::set_display(console::reset); diff --git a/llama.cpp b/llama.cpp index 85cb3140d..989d27b9d 100644 --- a/llama.cpp +++ b/llama.cpp @@ -17861,6 +17861,10 @@ bool llama_token_is_eog(const struct llama_model * model, llama_token token) { ); } +bool llama_token_is_control(const struct llama_model * model, llama_token token) { + return llama_is_control_token(model->vocab, token); +} + llama_token llama_token_bos(const struct llama_model * model) { return model->vocab.special_bos_id; } diff --git a/llama.h b/llama.h index 16cece5db..16676269d 100644 --- a/llama.h +++ b/llama.h @@ -823,6 +823,9 @@ extern "C" { // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) LLAMA_API bool llama_token_is_eog(const struct llama_model * model, llama_token token); + // Identify if Token Id is a control token or a render-able token + LLAMA_API bool llama_token_is_control(const struct llama_model * model, llama_token token); + // Special tokens LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence From 3cbd23ed88c03a27e1eb6090ac4a8186ca9ac29a Mon Sep 17 00:00:00 2001 From: Brian Date: Sat, 25 May 2024 19:30:42 +1000 Subject: [PATCH 136/137] labeler: added Apple Metal detector (+Kompute) (#7529) * labeler: added Apple Metal detector [no ci] * labeler: add Kompute to detector [no ci] --- .github/labeler.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index a67f78044..97d739b58 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,5 +1,16 @@ # https://github.com/actions/labeler - +Kompute: + - changed-files: + - any-glob-to-any-file: + - ggml-kompute.h + - ggml-kompute.cpp + - README-kompute.md +Apple Metal: + - changed-files: + - any-glob-to-any-file: + - ggml-metal.h + - ggml-metal.cpp + - README-metal.md SYCL: - changed-files: - any-glob-to-any-file: @@ -9,6 +20,7 @@ SYCL: Nvidia GPU: - changed-files: - any-glob-to-any-file: + - ggml-cuda.h - ggml-cuda/** Vulkan: - changed-files: From 9588f196b1d7b21bdff013fcf958c249576b2619 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 25 May 2024 15:21:30 +0300 Subject: [PATCH 137/137] train : change default FA argument (#7528) --- common/train.cpp | 2 +- examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/train.cpp b/common/train.cpp index 2d41a1d29..fef1e57c9 100644 --- a/common/train.cpp +++ b/common/train.cpp @@ -1052,7 +1052,7 @@ struct train_params_common get_default_train_params_common() { params.custom_n_ctx = false; - params.use_flash = true; + params.use_flash = false; params.use_checkpointing = true; params.sample_start = ""; diff --git a/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp b/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp index 746c3fbef..8ca9f8915 100644 --- a/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp +++ b/examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp @@ -774,7 +774,7 @@ static struct train_params get_default_train_params() { params.samples_start_after_nl = false; params.use_adam = true; - params.use_flash = true; + params.use_flash = false; params.use_scratch = true; // only adam