o ZhO @sddlZddlmZmZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZddlmZe r9ddlZeeZd d d d d ddde de dide didZeedZGdddeZGdddZGdddeZGdddeZGdddeZGdd d eZGd!d"d"eZGd#d$d$eZ Gd%d&d&eZ!Gd'd(d(eZ"eeeeeee e!e"e"d) Z#d*d+Z$   ,d5d-ee%d.ee&d/e%fd0d1Z'd6d3d4Z(dS)7N) NamedTupleOptional)tqdm)GGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPING_gguf_parse_value)is_torch_available)is_gguf_available) get_loggerversion tensor_countkv_count)r r r file_typequantization_version)rr)ZGGUFZgeneral tokenizertokenizer_config)ignoreconfigrrrc@s(eZdZUejed<eed<eed<dS) GGUFTensorweightsnamemetadataN)__name__ __module__ __qualname__npndarray__annotations__strdictr!r!W/var/www/auris/lib/python3.10/site-packages/transformers/modeling_gguf_pytorch_utils.pyr7s  rc@seZdZdddZddZdS)TensorProcessorNcCs|pi|_dSNrselfrr!r!r"__init__>szTensorProcessor.__init__cKs t||iSr$rr'rrkwargsr!r!r"processAs zTensorProcessor.processr$)rrrr(r,r!r!r!r"r#=s  r#c sLeZdZd fdd ZddZ d dejdedeed ejfd d Z Z S) LlamaTensorProcessorNctj|ddSNr%superr(r& __class__r!r"r(FzLlamaTensorProcessor.__init__cKszd|vsd|vr7|jd}|jd}d||fvr t||iSd|vr,||||}n d|vr7||||}t||iS)Nz.attn_k.z.attn_q.Znum_attention_headsZnum_key_value_heads)rgetr_reverse_permute_weights)r'rrr+ num_heads num_kv_headsr!r!r"r,Is     zLlamaTensorProcessor.processrn_headr8returncCsZ|dur ||kr |}|jd|d}|j||dg|jddR}|dd|jS)Nrr)shapereshapeZswapaxes)r'rr9r8dimwr!r!r"r6Vs  z-LlamaTensorProcessor._reverse_permute_weightsr$) rrrr(r,rrintrr6 __classcell__r!r!r2r"r-Esr-csJeZdZd fdd ZddZdejdeeefded efd d Z Z S) Qwen2MoeTensorProcessorNcr.r/r0r&r2r!r"r(dr4z Qwen2MoeTensorProcessor.__init__cKs^d|vr|d}|d}|r|||||t|diSd|vr)tj|dd}t||iS)N_exptensor_key_mappingparsed_parametersZffn_gate_inp_shexprZaxis)r5_split_moe_expert_tensorrr expand_dims)r'rrr+rDrEr!r!r"r,gs    zQwen2MoeTensorProcessor.processrrErrDc Cs^||}|jdd}td|D]}|dd|d}||}tt||d|<qdS)NZ num_experts<r mlp.experts..tensors)rr5rangereplacetorch from_numpyrcopy) r'rrErrDZ w_counteriZ temp_nameZ exp_weightr!r!r"rGtsz0Qwen2MoeTensorProcessor._split_moe_expert_tensorr$) rrrr(r,rrr rrGrAr!r!r2r"rBcs  rBcsVeZdZd fdd ZddZdejdedefd d Zdejdedefd d Z Z S)BloomTensorProcessorNcr.r/r0r&r2r!r"r(r4zBloomTensorProcessor.__init__cKsNd|vr!|jd}|jd}d|vr||||}n||||}t||iS)NZattn_qkvr9Z hidden_sizeweight)r_reverse_reshape_weights_reverse_reshape_biasr)r'rrr+r7n_embedr!r!r"r,s   zBloomTensorProcessor.processrr9rWcCsxtj|ddd\}}}|||||}|||||}|||||}tj|||gdd}||d|||S)NrrFr)r array_splitr=stack)r'rr9rWqkvZ qkv_weightsr!r!r"rUs z-BloomTensorProcessor._reverse_reshape_weightscCs^t|d\}}}||||}||||}||||}tj|||gdd}|S)NrXrrF)rrYr=rZflatten)r'rr9rWZq_biasZk_biasZv_biasqkv_biasr!r!r"rVs z*BloomTensorProcessor._reverse_reshape_biasr$) rrrr(r,rrr@rUrVrAr!r!r2r"rSs  rSc&eZdZdfdd ZddZZS)T5TensorProcessorNcr.r/r0r&r2r!r"r(r4zT5TensorProcessor.__init__cKs8d}|dD] }|rt|}nqt||d|iS)NrKbid)splitisdigitr@r)r'rrr+rbchunkr!r!r"r,szT5TensorProcessor.processr$rrrr(r,rAr!r!r2r"raracr`)GPT2TensorProcessorNcr.r/r0r&r2r!r"r(r4zGPT2TensorProcessor.__init__cKsfd|vsd|vsd|vsd|vr|j}|dkr-d}|di}tt||d|<d}t||iS) Nzattn_qkv.weightzffn_down.weightz ffn_up.weightzattn_output.weight output.weightzlm_head.weightrErL)Tr5rOrPrrQr)r'rrr+rEr!r!r"r,s  zGPT2TensorProcessor.processr$rfr!r!r2r"rhrgrhcr`)MambaTensorProcessorNcr.r/r0r&r2r!r"r(r4zMambaTensorProcessor.__init__cKs6d|vr tj|dd}d|vrt| }t||iS)Nzssm_conv1d.weightrrFZssm_a)rrHlogrr*r!r!r"r,s   zMambaTensorProcessor.processr$rfr!r!r2r"rkrgrkcr`)NemotronTensorProcessorNcr.r/r0r&r2r!r"r(r4z NemotronTensorProcessor.__init__cKd|vr|d}t||iSNz norm.weightrr)r*r!r!r"r, zNemotronTensorProcessor.processr$rfr!r!r2r"rmsrmcr`)Gemma2TensorProcessorNcr.r/r0r&r2r!r"r(r4zGemma2TensorProcessor.__init__cKrnror)r*r!r!r"r,rpzGemma2TensorProcessor.processr$rfr!r!r2r"rqsrq) llamaqwen2moebloomt5 t5encoderZgpt2ZmambaZnemotronZgemma2gemma3cs,||jvrgS|j|fddjDS)Ncg|] }tj|jqSr!rpartstypes.0Z _data_indexvaluer!r" zread_field..)fieldsdata)readerfieldr!r~r" read_fields  r model_type num_layers qual_namecstrtrddlm}m}n tdtd|dur |jj n|}|dur*|jj n|}|dkr3d}n |dkr:d }n|d kr@d }d}| D] \}}||krR|}nqF|dur_t d |d |||} i| } | D]>} |d krd| vrtdd| } | d} } | ds| dr| dd\} } d| } | | }|durqn|| || <qn|}r|D]!\} }t||||| dd}fdd| D}|qS)aY GGUF uses this naming convention for their tensors from HF checkpoint: `blk.N.BB.weight` and `blk.N.BB.bias` where N signifies the block number of a layer, and BB signifies the attention/mlp layer components. See "Standardized tensor names" in https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details. r)MODEL_ARCH_NAMESget_tensor_name_mapLoading a GGUF checkpoint in PyTorch, requires both PyTorch and GGUF>=0.10.0 to be installed. Please see https://pytorch.org/ and https://github.com/ggerganov/llama.cpp/tree/master/gguf-py for installation instructions.KPlease install torch and gguf>=0.10.0 to load a GGUF checkpoint in PyTorch.NZcoherez command-r qwen2_moers gemma3_textrwzUnknown gguf model_type: z in gguf-py. This might because you're using an outdated version of gguf-py package, you can install `gguf` package from source refer to https://github.com/ggerganov/llama.cpp/tree/master/gguf-py#developmentrJzmlp.experts.\d+.rz.weightz.biasrKr)rcsi|] \}}|vr||qSr!r!)r}r\r]Zgguf_to_hf_name_mapr!r" Trz+get_gguf_hf_weights_map..)r r ggufrrloggererror ImportErrorrrZnum_hidden_layersitemsNotImplementedError state_dictkeysresubendswithrsplitget_namenamed_childrenget_gguf_hf_weights_mapupdate)Zhf_modelrrrrrarchkeyrZname_maprZhf_namersuffixZ gguf_namerchildZsub_mapr!rr"r sX         rFc strtrddlm}m}n tdtd||}|j}t | }ddt D}t |dd} t |d} d } d | vrEd | vrEd } nd | vsMd | vrad|dd<d | vr^dg|dd<d } n| } d| vrid} d| vrhddt fdd|jD} t fdd|jD} | |dd<| |dd<| tvr| tvrtd| ddd g}td!d|jDp| |v|dd"<|jD]\}|| | }|d#}|d}d#|d$d }fd%d&jD}t|d$kr|d}t|tr| |vr|| | }t D]5}t |}||vr:|||vr:|||}|d'kr%q|d ur0||||<||vr:||q||vrLtd(|d)|q|dd*d+kr\d,|dd*<d-|dvr||d.}d/|vrwt|d/|dd-<ntd0|ri|d1<t|}|di}t| t }||d2}t!|jd3d4D]3}|j"}||j|j#}|j$||||d5}|j%}|j"}||vrq||}t&'t()||d1|<qt|dkrtd6||S)7a Load a GGUF file and return a dictionary of parsed parameters containing tensors, the parsed tokenizer and config attributes. Args: gguf_checkpoint_path (`str`): The path the to GGUF file to load return_tensors (`bool`, defaults to `False`): Whether to read the tensors from the file and return them. Not doing so is faster and only loads the metadata in memory. r) GGUFReader dequantizerrcSsi|]}|iqSr!r!)r}r\r!r!r"rssz(load_gguf_checkpoint..zgeneral.architecturez general.nameNrrZmistralrurvTrZ is_gated_actZT5EncoderModelZ architecturesrsrZstablelm>z attn_v.biasz attn_q.biasz attn_k.biasZffn_normc3s$|] }D]}||jvVqqdSr$r)r}tensorZ bias_name)attn_bias_namer!r" s"z'load_gguf_checkpoint..c3s|]}|jvVqdSr$rr}r) ffn_norm_namer!r"rZ use_qkv_biasuse_parallel_residualzGGUF model with architecture z is not supported yet.Zfalconrtcss|]}d|jkVqdS)riNrrr!r!r"rrZtie_word_embeddingsrKrcrxr!ryr|)rr!r"rrz(load_gguf_checkpoint..z1Some keys were not parsed and added into account z | rrwrZ vocab_sizertokenszCan't find a way to retrieve missing config vocab_size from tokenizer parameters. This will use default value from model config class and cause unexpected behavior.rLr%z,Converting and de-quantizing GGUF tensors...)Zdesc)rrrDrEz0Some keys of the GGUF file were not considered: )*r r rrrrrrrlistrGGUF_TO_TRANSFORMERS_MAPPINGranyrLGGUF_SUPPORTED_ARCHITECTURES ValueErrorallrrNrcjoinrlen isinstancerremoveinfowarningrr5TENSOR_PROCESSORSr#rrZ tensor_typer,rrOrPrrQ) Zgguf_checkpoint_pathZreturn_tensorsZ model_to_loadrrrrZ reader_keysrE architectureZ model_nameZupdated_architecturer_r exceptionsZgguf_keyrcprefixZ config_keyrZ parameterZparameter_renamesZrenamed_config_keyZtokenizer_parametersrDrZProcessorClass processorrrrresultr!)rrrr"load_gguf_checkpointZs                       r)NNr)FN))rtypingrrnumpyrZ tqdm.autorZ integrationsrrrutilsr Zutils.import_utilsr Z utils.loggingr rOrrrrrrrr#r-rBrSrarhrkrmrqrrrr@rrr!r!r!r"sn      '     N