o Zh+@sdZddlZddlmZddlmZddlmZmZm Z m Z ddl Z ddl Z ddl m Z ddlmZdd lmZdd lmZmZmZmZmZmZd d lmZeeZdad dZGddde j j!Z"d)ddZ#d)ddZ$Gddde j%Z&Gddde j%Z'Gddde j%Z(eGdddeZ)eGdddeZ*eGd d!d!eZ+eGd"d#d#e)Z,ed$d%Gd&d'd'e)eZ-gd(Z.dS)*zPyTorch RWKV model.N) dataclass)Path)ListOptionalTupleUnion)nn)GenerationMixin)PreTrainedModel) ModelOutputauto_docstringis_bitsandbytes_availableis_ninja_availableis_torch_cuda_availablelogging) RwkvConfigcsddlm}ttjjjddfdddD}tdur'tj|kr'dSt d|d d d d d ddd|g}|d||t t j k|da|t_dS)Nr)loadZkernelsrwkvcsg|]}|qSr).0fZ kernel_folderrU/var/www/auris/lib/python3.10/site-packages/transformers/models/rwkv/modeling_rwkv.py 4z(load_wkv_cuda_kernel..)z wkv_op.cppz wkv_cuda.cuzwkv_cuda_bf16.cuz2Loading CUDA kernel for RWKV at context length of .z -res-usagez--maxrregcount 60z--use_fast_mathz-O3z -Xptxas -O3z--extra-device-vectorizationz-DTmax=Zwkv_)namesourcesverboseZextra_cuda_cflags) Ztorch.utils.cpp_extensionrr__file__resolveparentrwkv_cuda_kernelmax_seq_lengthloggerinforZ get_verbosityDEBUG)context_lengthZ load_kernelZcuda_kernel_filesflagsrrrload_wkv_cuda_kernel.s*    r+c@s(eZdZedddZedddZdS) RwkvLinearAttentionNFc Cs|\}}} |tjkrtd|dtjd|| t| ddkr4td|d| dt| dd |j|_|jjd ksP|jjd ksP|jjd ksP|jjd krTtd t |  }|jt j krp| }| }| }| }| }| }t j|t jd } |s|dur|durt j|| d t j|jt jd}|dddddfd8<nt jdd|Ddd }|jt jkrtj} ntj} | ||||| |n|jt jkrtjntj} | ||||| |||||| |durddt j|d ddD}| |j|fS)NzCannot process a batch with z+ tokens at the same time, use a maximum of z with this model. rzThe product of batch size (z) and hidden size (z") needs to be a round multiple of rcudazUCalling the CUDA kernel for wkv attention requires all tensors to be on CUDA devices. memory_formatr )dtypedevicer0籡*GcSg|]}|dqSr3Z unsqueezersrrrr}z/RwkvLinearAttention.forward..)dimcSr5r6)Zsqueezer8rrrrr:)sizer$r% ValueErrorminr1 input_dtyper2typetorchexpfloat contiguousfloat16 empty_likecontiguous_formatzerosfloat32catbfloat16Zforward_with_state_bf16Zforward_with_stateZ forward_bf16forwardZsave_for_backwardchunkto) ctx time_decay time_firstkeyvaluestate return_stateZ batch_sizeZseq_len hidden_sizeoutputZ forward_funcrrrrLOsd         zRwkvLinearAttention.forwardc Cs|j}|j\}}}}}tj|tj|tjkrtjntjd} tj|tjd} tj|tjd} tj|tjd} |tjkr>|}|tjkrFt j nt j } | |||||| | | | | | || || || |ddfS)N)r0r1r/)r?Z saved_tensorsrArFrGrKrIrErCr$Z backward_bf16backwardrDrN)rOZg_outputZg_stater?rPrQrRrSrWZ g_time_decayZ g_time_firstZg_keyZg_valueZ backward_funcrrrrXs@ zRwkvLinearAttention.backwardNFN)__name__ __module__ __qualname__ staticmethodrLrXrrrrr,Ns  >r,FcCs|\}}}t|}|dur=tj|dddftjd} tj|dddftjd} tj|dddftjdd} n|\} } } t| }t|D]p} |dd| f} |dd| f}t| | |}t| |}t| ||}|| ||}|| |}|||j |dd| f<t| || }t| ||}t| |}|| ||} || |} |} qL|s|dur| | | g}||fS)Nr)r1r4) r<rAZ zeros_likerIrBrangerCmaximumrNr1)rPrQrRrSrTrU_Z seq_lengthrWZ num_stateZ den_stateZ max_stateZ current_index current_key current_valueZmax_for_outpute1e2 numerator denominatorZ max_for_staterrrrwkv_linear_attention_cpus4 "       rhcCs`tdd||||fD}|ddk}tdus|s|r&t||||||dSt||||||S)Ncss|] }|jjdkVqdS)r.N)r2r@)rtrrr z(rwkv_linear_attention..rrTrU)anyr<r$rhr,apply)rPrQrRrSrTrUZno_cudaZ one_tokenrrrrwkv_linear_attentions rocs2eZdZd fdd Zd ddZd dd ZZS) RwkvSelfAttentionrcsDt||_tduotj|jk}tr0tr0|s0zt|jWnt y/t dYnw||_ |j }|jdur>|jn|}||_tt||_tt||_ttdd||_ttdd||_ttdd||_td|_tj||dd|_tj||dd|_tj||dd|_tj||dd|_dS)Nz9Could not load the custom CUDA kernel for RWKV attention.rrrrFZbias)super__init__configr$r%r)rrr+ Exceptionr&r'layer_idrVattention_hidden_sizer ParameterrAemptyrPrQ time_mix_keytime_mix_valuetime_mix_receptance ZeroPad2d time_shiftLinearrRrS receptancerW)selfrvrxZ kernel_loadedrVry __class__rrrus0   zRwkvSelfAttention.__init__NcCs|ddkr|dur|ddddd|jf}n||}|dur7|ddddd|jf|dddf<||j|d|j}||j|d|j}||j|d|j}||}||}t | |}|dur|dddf|ddddd|jf<||||fSNrrrr) r<rxrr|r}r~rRrSrAsigmoidr)rhiddenrTshiftedrRrSrrrrextract_key_values (  ( z#RwkvSelfAttention.extract_key_valueFc sj||d\}}}}|durtfdd|ddDnd}tjj||||d\}}|durb|d|dddddjf<|d|dddddjf<|d|d ddddjf<|||fS) NrTc3s(|]}|ddddjfVqdSrZrxr8rrrrj#s&z,RwkvSelfAttention.forward..r3rlrrr )rtuplerorPrQrxrW) rrrT use_cacherrRrSZ layer_staterrrrrL!s*    zRwkvSelfAttention.forwardrrZrY)r[r\r]rurrL __classcell__rrrrrps rpcs(eZdZdfdd ZdddZZS) RwkvFeedForwardrcst||_||_|j}|jdur|jnd|j}td|_t t dd||_ t t dd||_ tj||dd|_tj||dd|_tj||dd|_dS)NrrqrFrs)rtrurvrxrVintermediate_sizerrrrzrAr{r|r~rrRrrS)rrvrxrVrrrrru6s  zRwkvFeedForward.__init__NcCs |ddkr|dur|ddddd|jf}n||}|dur7|ddddd|jf|dddf<||j|d|j}||j|d|j}tt||}| |}t | |}|dur|dddf|ddddd|jf<|||fSr) r<rxrr|r~rAZsquareZrelurRrSrr)rrrTrrRrrSrrrrLGs ( ( zRwkvFeedForward.forwardrrZr[r\r]rurLrrrrrr5srcs&eZdZfddZdddZZS) RwkvBlockcsvt||_||_|dkrtj|j|jd|_tj|j|jd|_ tj|j|jd|_ t |||_ t |||_dS)Nr)Zeps)rtrurvrxr LayerNormrVZlayer_norm_epsilonpre_lnln1ln2rp attentionr feed_forward)rrvrxrrrru\s  zRwkvBlock.__init__NFcCs||jdkr ||}|j||||d\}}||}|j|||d\}}||}||f}|r8||f7}|S|d7}|S)Nr)rTrrrZ)rxrrrrr)rrrTroutput_attentionsrrZoutputsrrrrLjs   zRwkvBlock.forward)NFFrrrrrr[s rc@s2eZdZeZdZdgZddgZdZdZ ddZ dS) RwkvPreTrainedModelrrrPrQTcs"t|tr|j}|jj}|jj|j||dd||}tjfddt D|j j |j j d}|ddddf}fddt D}tj||j j |j j d}tjddt D|jj |jj dd }t:||j _t|jtd ||j_t|||j _t||d |j_t|d ||j_WddS1swYdSt|tr|j}|jj}|jjd||}tjfd dt D|j j |j j d}|ddddf}tt|||j _t|||j_WddS1swYdSdS) zInitialize the weights.rg?cg|]}|qSrrrirVrrrrz5RwkvPreTrainedModel._init_weights..r1r2Ncs,g|]}dd|dddqS)rgffffff?g?r)rh)ry ratio_0_to_1rrrscSsg|] }|dddqS)rr rrrrrrsg?g333333?crrrrrrrrr) isinstancerprxrvnum_hidden_layersrVryrAZtensorr_r|r1r2rPrQno_graddataZ ones_likemathlogpowr}r~r)rmodulerxrZratio_1_to_almost0Z time_weightZ decay_speedZzigzagr)ryrVrr _init_weightss`     "   $z!RwkvPreTrainedModel._init_weightsN) r[r\r]rZ config_classZbase_model_prefixZ_no_split_modulesZ_keep_in_fp32_modulesZsupports_gradient_checkpointingZ _is_statefulrrrrrr}s rc@sneZdZUdZdZeejed<dZ ee ejed<dZ ee ejdfed<dZ ee ejdfed<dS) RwkvOutputa Class for the RWKV model outputs. Args: last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nlast_hidden_staterT. hidden_states attentions)r[r\r]__doc__rrrA FloatTensor__annotations__rTrrrrrrrrrs rc@seZdZUdZdZeejed<dZ eejed<dZ ee ejed<dZ ee ejdfed<dZee ejdfed<dS) RwkvCausalLMOutputa| Base class for causal language model (or autoregressive) outputs. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. NlosslogitsrT.rr)r[r\r]rrrrArrrrTrrrrrrrrrs rcseZdZfddZddZddZe        ddeej d eej d eej d ee ej d ee d ee dee dee de eeffddZddZddZZS) RwkvModelcsdttjj|_tfddtj D|_ t j|_ d|_ d|_|dS)Ncsg|]}t|dqS)r)r)ridxrvrrrsz&RwkvModel.__init__..F)rtrurZ Embedding vocab_sizerV embeddingsZ ModuleListr_rblocksrln_outlayers_are_rescaledgradient_checkpointing post_initrrvrrrrus   zRwkvModel.__init__cC|jSrZrrrrrget_input_embeddingszRwkvModel.get_input_embeddingscC ||_dSrZrrZnew_embeddingsrrrset_input_embeddings zRwkvModel.set_input_embeddingsN input_idsattention_mask inputs_embedsrTrroutput_hidden_states return_dictreturnc s,|dur|n|jj}|dur|n|jj}|dur|n|js!|jjnd}|dur)|n|jj}|dur6td|j|jkr@| |durLdurLt d|durXdurXt ddura| ||r|dur d|jj |jjffddtd D}|d d 8<|jr|jr|rtd d}} |rd nd} |rd nd} t|jD]H\} } |jr|jr|| j| |||\} }}n | | |||d\} }}|jr|jjdkr| d|jjdkr| d} |r| | f} |r| |f} q|| } |r| | f} |stdd| || | fDSt| || | dS)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. NFz<`attention_mask` was passed, but it is unused in this model.zDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrcs0g|]}tj|dkrjntjjdqS)rr)rArHr1rIr2rrshaperrrKs z%RwkvModel.forward..rgꌠ9Y>)FzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...r)rTrrrr3css|] }|dur|VqdSrZr)rxrrrrj{rkz$RwkvModel.forward..)rrTrr)rvrrtrainingruse_return_dictr&Z warning_oncer_rescale_layersr=rr<rVrr_r enumeraterZ_gradient_checkpointing_func__call__ rescale_everyrrr)rrrrrTrrrrrZall_self_attentionsZall_hidden_statesrblockrrrrrLsv                 zRwkvModel.forwardcCsx|j|j kr dS|jjdkrtt|jD]\}}|jrA|jj j dt ||jj|j jj dt ||jjqt|jj j drl|jj j jdt ||jj|j jj jdt ||jjqt|jj j dr||jj |||j j|q|jj j dt ||jj|j jj dt ||jjqWdn1swY|j |_dS)Nrr3SCB quant_state)rrrvrrArrrrrWweightZmul_intrrShasattrrdiv_ _bnb_4bit_dequantize_and_rescale)rblock_idrrrrrs&   ""$ "zRwkvModel._rescale_layerscCsttstdddl}|j|jj|jj}|dt ||j j |j j |ddd|j}t|d|dS) z Perform the dequantization and rescaling of the weights of a given layer. After that operation the layer will be quantized again. z/Please install bitsandbytes to use this method.rNr3cpuF)Z requires_gradr)r ImportErrorZ bitsandbytesZ functionalZdequantize_4bitrrrrrrvrrZ Params4bitrNr2setattr)rZ target_layerrZbnbZdequant_weightsZ quant_weightrrrrsz*RwkvModel._bnb_4bit_dequantize_and_rescale)NNNNNNNN)r[r\r]rurrr rrA LongTensorrrboolrrrrLrrrrrrrrsD    nrz The RWKV Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). )Z custom_introcseZdZdgZfddZddZddZdd d Ze         dd e e j d e e j d e e j de e e j de e j de ede ede ede edeeeffddZZS)RwkvForCausalLMz head.weightcs8t|t||_tj|j|jdd|_| dS)NFrs) rtrurrrrrVrheadrrrrrrus   zRwkvForCausalLM.__init__cCrrZrrrrrget_output_embeddingsrz%RwkvForCausalLM.get_output_embeddingscCrrZrrrrrset_output_embeddingsrz%RwkvForCausalLM.set_output_embeddingsNcKsT|dur|dddfd}|dur|durd|i}nd|i}||d<||d<|S)NrrrrrTrr7)rrrTrrkwargsZ model_inputsrrrprepare_inputs_for_generations z-RwkvForCausalLM.prepare_inputs_for_generationrrrrTlabelsrrrrrc  Ks| dur| n|jj} |j||||||| d} | d} || } d}|dur3|j| |fd|jji| }| sI| f| dd}|durG|f|S|St|| | j| j| j dS)aI input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. N)rrTrrrrrrr)rrrTrr) rvrrrZ loss_functionrrrTrr)rrrrrTrrrrrrZ rwkv_outputsrrrrWrrrrLs@% zRwkvForCausalLM.forward)NNN) NNNNNNNNN)r[r\r]Z_tied_weights_keysrurrrr rrArrrrrrrrLrrrrrrsJ      r)rrrrY)/rr dataclassesrpathlibrtypingrrrrrAZtorch.utils.checkpointrZ generationr Zmodeling_utilsr utilsr r rrrrZconfiguration_rwkvrZ get_loggerr[r&r$r+ZautogradFunctionr,rhroModulerprrrrrrr__all__rrrrsF         j , F&"B 3 l