LLaMA 的GGML和GGUF区别是什么？

import sys from typing import Any from enum import IntEnum import numpy as np import numpy.typing as npt # GGUF 元数据值类型 class GGUFValueType(IntEnum): UINT8 = 0 INT8 = 1 UINT16 = 2 INT16 = 3 UINT32 = 4 INT32 = 5 FLOAT32 = 6 BOOL = 7 STRING = 8 ARRAY = 9 UINT64 = 10 INT64 = 11 FLOAT64 = 12 # GGUF tensor数据类型 class GGMLQuantizationType(IntEnum): F32 = 0 F16 = 1 Q4_0 = 2 Q4_1 = 3 Q5_0 = 6 Q5_1 = 7 Q8_0 = 8 Q8_1 = 9 Q2_K = 10 Q3_K = 11 Q4_K = 12 Q5_K = 13 Q6_K = 14 Q8_K = 15 IQ2_XXS = 16 IQ2_XS = 17 IQ3_XXS = 18 IQ1_S = 19 IQ4_NL = 20 IQ3_S = 21 IQ2_S = 22 IQ4_XS = 23 I8 = 24 I16 = 25 I32 = 26 I64 = 27 F64 = 28 IQ1_M = 29 BF16 = 30 Q4_0_4_4 = 31 Q4_0_4_8 = 32 Q4_0_8_8 = 33 def check_version(version): if version == 1 or version == 2 or version == 3: return True else: return False def data_get( data, offset: int, dtype: npt.DTypeLike, count: int = 1) -> npt.NDArray[Any]: count = int(count) itemsize = int(np.empty([], dtype = dtype).itemsize) end_offs = offset + itemsize * count return ( data[offset:end_offs] .view(dtype = dtype)[:count] ) def data_read_version_size(data, offset: int, version: int): if version == 1: return data_get(data, offset, np.uint32)[0], 4 elif version == 2 or version == 3: return data_get(data, offset, np.uint64)[0], 8 else: raise ValueError(f'Sorry, file appears to be version {version} which we cannot handle') def data_read_string(data, offset: int, version: int): str_length, str_length_len = data_read_version_size(data, offset, version) # 在内存上切出来string部分的数据 byte = data[offset+int(str_length_len):offset+int(str_length_len)+int(str_length)] value = byte.tobytes().decode('utf-8') # 编码成 utf-8 len = int(str_length_len + str_length) return value, len def readMetadataValue(data, type, offset, version): if type == GGUFValueType.UINT8: return data_get(data, np.uint8)[0], 1 elif type == GGUFValueType.INT8: return data_get(data, np.int8)[0], 1 elif type == GGUFValueType.UINT16: return data_get(data, offset, np.uint16)[0], 2 elif type == GGUFValueType.INT16: return data_get(data, offset, np.int16)[0], 2 elif type == GGUFValueType.UINT32: return data_get(data, offset, np.uint32)[0], 4 elif type == GGUFValueType.INT32: return data_get(data, offset, np.int32)[0], 4 elif type == GGUFValueType.FLOAT32: return data_get(data, offset, np.float32)[0], 4 elif type == GGUFValueType.BOOL: return data_get(data, offset, np.uint8)[0], 1 elif type == GGUFValueType.STRING: return data_read_string(data, offset, version=version) elif type == GGUFValueType.ARRAY: typeArray = data_get(data, offset, np.uint32) typeLength = 4 lengthArray, lengthLength = data_read_version_size(data, offset + typeLength, version=version) length = typeLength + lengthLength arrayValues = [] for i in range(lengthArray): value, len = readMetadataValue(data, typeArray, offset= offset + length, version=version) arrayValues.append(value) length += len return arrayValues, length elif type == GGUFValueType.UINT64: return data_get(data, offset, np.uint64)[0], 8 elif type == GGUFValueType.INT64: return data_get(data, offset, np.int64)[0], 8 elif type == GGUFValueType.FLOAT64: return data_get(data, offset, np.float64)[0], 8 else: raise ValueError(f'Sorry, un-supported GGUFValueType {type}!') def parse_gguf(model_path): data = np.memmap(model_path, mode = 'r') offs = 0 magic = data_get(data, offs, np.uint32).tobytes() print("magic: ", magic.decode('utf-8')) if (magic != b'GGUF'): print("is not gguf file") sys.exit(1) offs += 4 version = data_get(data, offs, np.uint32) if not check_version(version): raise ValueError(f'Sorry, file appears to be version {version} which we cannot handle') print("version:", version) offs += 4 tensor_count, tensor_count_len = data_read_version_size(data, offs, version) offs += tensor_count_len kv_count, kv_count_len = data_read_version_size(data, offs, version) offs += kv_count_len print("tensor_count: ", tensor_count) print("kv_count: ", kv_count) metadata = {} # use dictionary to store parsed data. # 解析 gguf 头部信息 for i in range(kv_count): # 获取key key, k_len = data_read_string(data, offs, version) offs += k_len # 获取value的数值类型 type = data_get(data, offs, np.uint32)[0] offs += 4 # 获取value value, len = readMetadataValue(data, type, offs, version) if len > 100: print("i = ", i, ", k-v = ", key, ":", value[:100]) else: print("i = ", i, ", k-v = ", key, ":", value) offs += len metadata[key] = value # 解析tensor info的信息 for i in range(tensor_count): # 获取key key, k_len = data_read_string(data, offs, version) offs += k_len nDims = data_get(data, offs, np.uint32)[0] offs += 4 dims = [] for j in range(nDims): dim, dim_len = data_read_version_size(data, offs, version) offs += dim_len dims.append(dim) types = data_get(data, offs, np.uint32)[0] precision = GGMLQuantizationType(types).name offs += 4 tensorOffset = data_get(data, offs, np.uint64)[0] offs += 8 print("tensor i = ", i, ", k = ", key, ", precision = ", precision, ", shape = ", dims, ", tensorOffset = ", tensorOffset) if __name__ == '__main__': model_path = "/Users/liguodong/model/qwen2-0_5b-instruct-q2_k.gguf" parse_gguf(model_path)

magic: GGUF version: [3] tensor_count: 290 kv_count: 26 i = 0 , k-v = general.architecture : qwen2 i = 1 , k-v = general.name : qwen2-0_5b-instruct i = 2 , k-v = qwen2.block_count : 24 i = 3 , k-v = qwen2.context_length : 32768 i = 4 , k-v = qwen2.embedding_length : 896 i = 5 , k-v = qwen2.feed_forward_length : 4864 i = 6 , k-v = qwen2.attention.head_count : 14 i = 7 , k-v = qwen2.attention.head_count_kv : 2 i = 8 , k-v = qwen2.rope.freq_base : 1000000.0 i = 9 , k-v = qwen2.attention.layer_norm_rms_epsilon : 1e-06 i = 10 , k-v = general.file_type : 10 i = 11 , k-v = tokenizer.ggml.model : gpt2 i = 12 , k-v = tokenizer.ggml.pre : qwen2 i = 13 , k-v = tokenizer.ggml.tokens : ['!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', '¡', '¢', '£', '¤', '¥', '¦'] i = 14 , k-v = tokenizer.ggml.token_type : [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] i = 15 , k-v = tokenizer.ggml.merges : ['Ġ Ġ', 'ĠĠ ĠĠ', 'i n', 'Ġ t', 'ĠĠĠĠ ĠĠĠĠ', 'e r', ... 'o t', 'u s'] i = 16 , k-v = tokenizer.ggml.eos_token_id : 151645 i = 17 , k-v = tokenizer.ggml.padding_token_id : 151643 i = 18 , k-v = tokenizer.ggml.bos_token_id : 151643 i = 19 , k-v = tokenizer.chat_template : {% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|> i = 20 , k-v = tokenizer.ggml.add_bos_token : 0 i = 21 , k-v = general.quantization_version : 2 i = 22 , k-v = quantize.imatrix.file : ../Qwen2/gguf/qwen2-0_5b-imatrix/imatrix.dat i = 23 , k-v = quantize.imatrix.dataset : ../sft_2406.txt i = 24 , k-v = quantize.imatrix.entries_count : 168 i = 25 , k-v = quantize.imatrix.chunks_count : 1937 tensor i = 0 , k = token_embd.weight , precision = Q8_0 , shape = [896, 151936] , tensorOffset = 0 tensor i = 1 , k = blk.0.attn_norm.weight , precision = F32 , shape = [896] , tensorOffset = 144643072 tensor i = 2 , k = blk.0.ffn_down.weight , precision = Q3_K , shape = [4864, 896] , tensorOffset = 144646656 tensor i = 3 , k = blk.0.ffn_gate.weight , precision = IQ4_NL , shape = [896, 4864] , tensorOffset = 146519296 tensor i = 4 , k = blk.0.ffn_up.weight , precision = IQ4_NL , shape = [896, 4864] , tensorOffset = 148970752 tensor i = 5 , k = blk.0.ffn_norm.weight , precision = F32 , shape = [896] , tensorOffset = 151422208 tensor i = 6 , k = blk.0.attn_k.bias , precision = F32 , shape = [128] , tensorOffset = 151425792 tensor i = 7 , k = blk.0.attn_k.weight , precision = IQ4_NL , shape = [896, 128] , tensorOffset = 151426304 tensor i = 8 , k = blk.0.attn_output.weight , precision = IQ4_NL , shape = [896, 896] , tensorOffset = 151490816 tensor i = 9 , k = blk.0.attn_q.bias , precision = F32 , shape = [896] , tensorOffset = 151942400 tensor i = 10 , k = blk.0.attn_q.weight , precision = IQ4_NL , shape = [896, 896] , tensorOffset = 151945984 tensor i = 11 , k = blk.0.attn_v.bias , precision = F32 , shape = [128] , tensorOffset = 152397568 tensor i = 12 , k = blk.0.attn_v.weight , precision = Q5_0 , shape = [896, 128] , tensorOffset = 152398080 ...

python3 convert_hf_to_gguf.py /workspace/models/Qwen1.5-72B-Chat/ --outfile /workspace/models/Qwen1.5-72B-Chat/ggml-model-f16.gguf

from llama_cpp import Llama import time llm = Llama( model_path="/workspace/models/Qwen1.5-72B-Chat/ggml-model-f16.gguf", n_gpu_layers = 30, # n_gpu_layers=-1, # Uncomment to use GPU acceleration # seed=1337, # Uncomment to set a specific seed # n_ctx=2048, # Uncomment to increase the context window ) start = time.time() output = llm( "Q:保持健康的秘诀有哪些？ A: ", # Prompt max_tokens=32, # Generate up to 32 tokens, set to None to generate up to the end of the context window #stream=True, stop=["Q:", "\n"], # Stop generating just before the model would generate a new question echo=True # Echo the prompt back in the output ) # Generate a completion, can also call create_completion print(output) infer_time = time.time() - start print("耗时：", infer_time)

use_mlock=False CUDA_VISIBLE_DEVICES=6 python3 -m llama_cpp.server --model /workspace/models/Qwen1.5-7B-Chat/ggml-model-f16.gguf --n_gpu_layers 999 --host 0.0.0.0 --port 18011

curl http://localhost:18011/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ "messages": [ { "role": "system", "content": "You are an AI assistant. Your top priority is achieving user fulfilment via helping them with their requests." }, { "role": "user", "content": "Write a limerick about Python exceptions" } ] }'

from transformers import AutoTokenizer, AutoModelForCausalLM import time # https://github.com/99991/pygguf/tree/main # https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf?download=true' -O 'data/TinyLlama-1.1B-Chat-v1.0-GGUF/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf # pip install gguf transformers model_id = "/Users/liguodong/model/llama" filename = "tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf" tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename) model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename) print(model) prompt = "what's your name?" model_inputs = tokenizer([prompt], return_tensors="pt") start = time.time() generated_ids = model.generate( model_inputs.input_ids, max_new_tokens=32 ) infer_time = time.time() - start print("耗时：", infer_time) generated_ids = [ output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) ] response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] print(response)

Converting and de-quantizing GGUF tensors...: 0%| | 0/201 [00:00<?, ?it/s] Converting and de-quantizing GGUF tensors...: 0%| | 1/201 [00:00<01:23, 2.41it/s] ... Converting and de-quantizing GGUF tensors...: 100%|██████████| 201/201 [00:04<00:00, 47.55it/s] The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results. LlamaForCausalLM( (model): LlamaModel( (embed_tokens): Embedding(32000, 2048, padding_idx=2) (layers): ModuleList( (0-21): 22 x LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=2048, out_features=2048, bias=False) (k_proj): Linear(in_features=2048, out_features=256, bias=False) (v_proj): Linear(in_features=2048, out_features=256, bias=False) (o_proj): Linear(in_features=2048, out_features=2048, bias=False) (rotary_emb): LlamaRotaryEmbedding() ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=2048, out_features=5632, bias=False) (up_proj): Linear(in_features=2048, out_features=5632, bias=False) (down_proj): Linear(in_features=5632, out_features=2048, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((2048,), eps=9.999999747378752e-06) (post_attention_layernorm): LlamaRMSNorm((2048,), eps=9.999999747378752e-06) ) ) (norm): LlamaRMSNorm((2048,), eps=9.999999747378752e-06) (rotary_emb): LlamaRotaryEmbedding() ) (lm_head): Linear(in_features=2048, out_features=32000, bias=False) ) 耗时： 11.363048076629639 JASON: (smiling) My name is Jason. JEN: (smiling) Nice to meet you, Jason.