o ZhG@sddlmZmZmZmZmZddlZddlmZddlm Z ddl m Z m Z m Z mZddlmZddlmZdd lmZdd lmZdd lmZdd lmZmZmZmZmZdd lm Z m!Z!ddl"m#Z#m$Z$ddl%m&Z&ddl'm(Z(m)Z)m*Z*m+Z+m,Z,ddl-m.Z.e+rddl/m0Z0ddl1m2Z2e,3e4Z5Gdddej6Z7ddZ8dDddZ9dej:de;dej:fddZ< dEd!ej6d"ej:d#ej:d$ej:d%eej:d&e=d'e=fd(d)Z>Gd*d+d+ej6Z?ed,Gd-d.d.ej6Z@Gd/d0d0eZAe)Gd1d2d2e$ZBGd3d4d4ej6ZCe)Gd5d6d6eBZDGd7d8d8ee(ZEe)Gd9d:d:eBeZFe)Gd;d<dGd?d@d@eBZHe)GdAdBdBeBZIgdCZJdS)F)CallableListOptionalTupleUnionN)nn)ACT2FN)Cache DynamicCacheSlidingWindowCache StaticCache)GenerationMixin)use_kernel_forward_from_hub)AttentionMaskConverter)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPastQuestionAnsweringModelOutput SequenceClassifierOutputWithPastTokenClassifierOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack) LossKwargsauto_docstringcan_return_tupleis_torch_flex_attn_availablelogging) MistralConfig) BlockMask)make_flex_block_causal_maskcs$eZdZfddZddZZS) MistralMLPcsrt||_|j|_|j|_tj|j|jdd|_tj|j|jdd|_tj|j|jdd|_ t |j |_ dSNFbias) super__init__config hidden_sizeZintermediate_sizerLinear gate_projup_proj down_projr Z hidden_actact_fnselfr, __class__[/var/www/auris/lib/python3.10/site-packages/transformers/models/mistral/modeling_mistral.pyr++s zMistralMLP.__init__cCs$||||||}|SN)r1r2r/r0)r4xr1r7r7r8forward5s zMistralMLP.forward)__name__ __module__ __qualname__r+r; __classcell__r7r7r5r8r&*s  r&cCsH|dd|jddf}|d|jdddf}tj| |fddS)z*Rotates half the hidden dims of the input..Ndim)shapetorchcat)r:x1Zx2r7r7r8 rotate_half:srHcCsD||}||}||t||}||t||}||fS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`, *optional*): Deprecated and unused. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. ) unsqueezerH)qkcossin position_idsZ unsqueeze_dimZq_embedZk_embedr7r7r8apply_rotary_pos_embAs  rO hidden_statesn_repreturncCs^|j\}}}}|dkr |S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) r"N)rDexpandreshape)rPrQbatchnum_key_value_headsslenhead_dimr7r7r8 repeat_kv\s 0rYmodulequerykeyvalueattention_maskscalingdropoutc Kst||j}t||j} t||dd|} |dur3|ddddddd|jdf} | | } tjj| dtj d |j } tjj | ||j d} t| | } | dd} | | fS)NrArr@)rCdtype)ptrainingr")rYnum_key_value_groupsrEmatmul transposerDrZ functionalZsoftmaxfloat32torcrare contiguous) r[r\r]r^r_r`rakwargs key_states value_states attn_weights causal_mask attn_outputr7r7r8eager_attention_forwardhs & rrcseZdZdZdedeffdd Z  ddejde ejejfd e ejd e e d e ej d e ed e eje eje e ejffddZZS)MistralAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr, layer_idxcst||_||_t|ddp|j|j|_|j|j|_ |jd|_ |j |_ d|_ t j|j|j|jdd|_t j|j|j|jdd|_t j|j|j|jdd|_t j|j|j|jdd|_dS)NrXgTFr()r*r+r,rtgetattrr-Znum_attention_headsrXrVrfr`attention_dropoutZ is_causalrr.q_projk_projv_projo_projr4r,rtr5r7r8r+s   zMistralAttention.__init__NrPposition_embeddingsr_past_key_valuecache_positionrlrRcKsT|jdd}g|d|jR}|||dd} |||dd} |||dd} |\} } t| | | | \} } |durW| | |d}|| | |j |\} } t }|j j dkrw|j j dkrq| ddrqtd nt|j j }||| | | |f|jsd n|j|jt|j d dd |\}}|jg|dR}||}||fS) Nr@r"rA)rMrLr~eagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.rZsliding_window)rar`r)rDrXrwviewrhrxryrOupdatertrrr,_attn_implementationgetlogger warning_oncerrervr`rurTrkrz)r4rPr|r_r}r~rlZ input_shapeZ hidden_shapeZ query_statesrmrnrLrMZ cache_kwargsZattention_interfacerqror7r7r8r;sB       zMistralAttention.forward)NN)r<r=r>__doc__r#intr+rETensorrrr LongTensorrrr;r?r7r7r5r8rss(rsZRMSNormcs.eZdZdfdd ZddZddZZS) MistralRMSNormư>cs&ttt||_||_dS)z= MistralRMSNorm is equivalent to T5LayerNorm N)r*r+r ParameterrEZonesweightvariance_epsilon)r4r-epsr5r7r8r+s  zMistralRMSNorm.__init__cCsJ|j}|tj}|djddd}|t||j}|j||S)NrAr@T)Zkeepdim) rcrjrEripowmeanZrsqrtrr)r4rPZ input_dtypeZvariancer7r7r8r;s  zMistralRMSNorm.forwardcCst|jjd|jS)Nz, eps=)tuplerrDrr4r7r7r8 extra_reprszMistralRMSNorm.extra_repr)r)r<r=r>r+r;rr?r7r7r5r8rsrcseZdZdedeffdd Z       ddejdeejd eej d ee d ee d ee d eej dee ejejfde ede ejee ejejfffddZZS)MistralDecoderLayerr,rtcsRt|j|_t||d|_t||_t|j|jd|_ t|j|jd|_ dS)N)r,rtr) r*r+r-rs self_attnr&mlpr rms_norm_epsinput_layernormpost_attention_layernormr{r5r7r8r+s  zMistralDecoderLayer.__init__NFrPr_rNr}r use_cacher~r|rlrRc Kst|} ||}|jd||||||||d| \}} | |}|} ||}||}| |}|f} |r8| | f7} | S)N)rPr_rNr}rrr~r|r7)rrrr) r4rPr_rNr}rrr~r|rlZresidualZself_attn_weightsoutputsr7r7r8r;s.      zMistralDecoderLayer.forward)NNNFFNN)r<r=r>r#rr+rErrrr boolrrr FloatTensorr;r?r7r7r5r8rs<    rc@sHeZdZeZdZdZdgZdgZdZ dZ dZ dZ dZ dZdZddZdS)MistralPreTrainedModelmodelTrpast_key_valuescCs|jj}t|tjr"|jjjd|d|jdur |jj dSdSt|tj rC|jjjd|d|j durA|jj|j  dSdSt|t rQ|jj ddSdS)NrZ)rstdg?)r,Zinitializer_range isinstancerr.rdataZnormal_r)Zzero_ Embedding padding_idxrZfill_)r4r[rr7r7r8 _init_weightss     z$MistralPreTrainedModel._init_weightsN)r<r=r>r#Z config_classbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesZ_skip_keys_device_placementZ_supports_flash_attn_2Z_supports_sdpaZ_supports_flex_attnZ_supports_cache_classZ_supports_quantized_cacheZ_supports_static_cacheZ_supports_attention_backendrr7r7r7r8rs rcs8eZdZddeffdd ZeeddZZ S)MistralRotaryEmbeddingNr,cstt|dr|jdur|jd|jd|_nd|_|j|_|j|_||_ t |j|_ | |j |\}|_ |j d|dd|j|_dS)N rope_scaling rope_typetypedefaultinv_freqF) persistent)r*r+hasattrrrrZmax_position_embeddingsZmax_seq_len_cachedZoriginal_max_seq_lenr,rZ rope_init_fnattention_scalingZregister_bufferrZoriginal_inv_freq)r4r,devicerr5r7r8r+,s   zMistralRotaryEmbedding.__init__c Cs|jddddf|jddd|j}|dddddf}t|jjtr6|jjdkr6|jjnd}t j |dd+|| dd}t j ||fdd }| |j}||j} Wdn1smwY|j|jd | j|jd fS) Nrr@r"ZmpscpuF) device_typeenabledrArB)rc)rfloatrSrDrjrrrstrrEZautocastrhrFrLrrMrc) r4r:rNZinv_freq_expandedZposition_ids_expandedrZfreqsZembrLrMr7r7r8r;=s0&zMistralRotaryEmbedding.forwardr9) r<r=r>r#r+rEZno_gradrr;r?r7r7r5r8r+s rcseZdZdeffdd ZddZddZee         d!d e e j d e e j d e e j d e e d e e jde ede ede ede e j deedefddZ d"d ee j dfde j de j d e def ddZed e j dedede jde j deded e fdd ZZS)# MistralModelr,cstj|_j|_tjj|j|_t fddt j D|_ t jjd|_td|_d|_|dS)Ncsg|]}t|qSr7)r).0rtr,r7r8 Vsz)MistralModel.__init__..rrF)r*r+ pad_token_idr vocab_sizerrr- embed_tokensZ ModuleListrangenum_hidden_layerslayersrrnormr rotary_embgradient_checkpointing post_initr3r5rr8r+Os   zMistralModel.__init__cC|jSr9rrr7r7r8get_input_embeddings_z!MistralModel.get_input_embeddingscC ||_dSr9rr4r^r7r7r8set_input_embeddingsb z!MistralModel.set_input_embeddingsN input_idsr_rNr inputs_embedsrroutput_hidden_statesr~flash_attn_kwargsrRc  Ks|dur|n|jj}|dur|n|jj}|dur|n|jj}|du|duAr*td|jr9|jr9|r9tdd}t |t dt fsFtd|durO| |}|rX|durXt }| durt|durd|nd} tj| | |jd|jd} |dur}| d}|||| ||} |} || |}|rdnd}|rdnd}|jd|jjD]&}|r|| f7}|| f| ||||| |d | }|d} |r||df7}q|| } |r|| f7}t| |r|nd||d S) Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.FzBThe `past_key_values` should be either a `Cache` object or `None`.rr"rr7)r_rNr}rrr~r|)last_hidden_staterrP attentions)r,rrr ValueErrorrrerrrrr rr get_seq_lengthrEarangerDrrI_update_causal_maskrrrrr)r4rr_rNrrrrrr~rpast_seen_tokensrprPr|Zall_hidden_statesZall_self_attnsZ decoder_layerZ layer_outputsr7r7r8r;esx             zMistralModel.forwardFr$ input_tensorc Cs|jjdkr2|dur&|dur&|dddf|dk}|r&td|dur0d|vr0|SdS|jjdkrDt|tjrBt |}|S|durL| nd}t|t }t|t } |jjdkrs|ss| ss|sst j||||jj|jdrsdS|j} t| j} |jd } | s|r|} nt|tjr|jdn|| d } |j|| | | ||jd|j|d }|jjdkr|dur|jjd vr|st || }|S) NZflash_attention_2r@rzYou are attempting to perform batched generation with padding_side='right' this may lead to unexpected behaviour for Flash Attention version of Mistral. Make sure to call `tokenizer.padding_side = 'left'` before tokenizing the input. rZZflex_attentionr)rZpast_key_values_lengthrZ is_trainingr")sequence_length target_lengthrcr~ batch_sizer,r)cudaZxpuZnpu)r,rsumitemsizerrrErr%rr r rZ_ignore_causal_mask_sdparrercfinfominrDZget_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionrrZ_unmask_unattended)r4r_rr~rrZis_padding_rightrZusing_static_cacheZusing_sliding_window_cacherc min_dtyperrrpr7r7r8rst $             z MistralModel._update_causal_maskrrrcrcCs|dur|dkr|}|St|j} tj||f| ||jd}tj||jd|ddk} |} t | ddr\| j dur\t |t rF||kr\tj||jd|dd| j k} | | || 9}|ddddddf|ddd}|dur|}|jd|kr|ddd|f}|jd} |ddddddd| f|ddddddf|j}|d k}|ddddddd| f|| |ddddddd| f<|S) a Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. config (`MistralConfig`): The model's configuration class past_key_values (`Cache`): The cache class that is being used currently to generate N)Z fill_valuercrrr@r"Zuse_sliding_windowTr)rCrErrfullrrrTZget_text_configrurrr Z bitwise_or_rSclonerDrjZ masked_fill)r_rrrcr~rr,rrprZdiagonal_attend_maskZ text_configZsliding_attend_maskZ mask_lengthZ padding_maskr7r7r8rs@!  $ 6  zBMistralModel._prepare_4d_causal_attention_mask_with_cache_position NNNNNNNNN)F)r<r=r>r#r+rrrrrrErrr rrrrrr;rr staticmethodrrcrr?r7r7r5r8rMs    d  Vrc@s eZdZdS)KwargsForCausalLMN)r<r=r>r7r7r7r8ras rcs eZdZdgZddiZddgdgfiZfddZdd Zd d Zd d Z ddZ ddZ ddZ e e           d%deejdeejdeejdeedeejdeejdeedeedeedeejd eeejfd!eed"efd#d$ZZS)&MistralForCausalLMzlm_head.weightlm_headZ colwise_reprPlogitscs@t|t||_|j|_tj|j|jdd|_| dSr') r*r+rrrrr.r-rrr3r5r7r8r+js   zMistralForCausalLM.__init__cC|jjSr9rrrr7r7r8rsz'MistralForCausalLM.get_input_embeddingscC ||j_dSr9rrr7r7r8rv z'MistralForCausalLM.set_input_embeddingscCrr9rrr7r7r8get_output_embeddingsyrz(MistralForCausalLM.get_output_embeddingscCrr9r)r4Znew_embeddingsr7r7r8set_output_embeddings|rz(MistralForCausalLM.set_output_embeddingscCrr9r)r4decoderr7r7r8 set_decoderrzMistralForCausalLM.set_decodercCrr9rrr7r7r8 get_decoderrzMistralForCausalLM.get_decoderNrrr_rNrrlabelsrrrr~logits_to_keeprlrRc  Ks|dur|n|jj}| dur| n|jj} |jd|||||||| | d | } | j}t| tr4t| dn| }||dd|ddf}d}|durX|j d|||jj d| }t ||| j | j | jdS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, MistralForCausalLM >>> model = MistralForCausalLM.from_pretrained("meta-mistral/Mistral-2-7b-hf") >>> tokenizer = AutoTokenizer.from_pretrained("meta-mistral/Mistral-2-7b-hf") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) rr_rNrrrrrr~)rrrlossrrrPrr7)r,rrrrrrslicer loss_functionrrrrPr)r4rr_rNrrrrrrr~rrlrrPZ slice_indicesrrr7r7r8r;s:'  zMistralForCausalLM.forward) NNNNNNNNNNr)r<r=r>Z_tied_weights_keysZ_tp_planZ_pp_planr+rrrrrrrrrrErrr rrrrrrrr;r?r7r7r5r8rdsf       rceZdZfddZddZddZee         ddee j d ee j d ee j d ee d ee j d ee j deedeedeedefddZZS)MistralForTokenClassificationcs|t||j|_t||_t|dddur|j}nt|dddur'|j}nd}t ||_ t |j |j|_ |dS)Nclassifier_dropouthidden_dropoutg?)r*r+ num_labelsrrrurrrZDropoutrar.r-scorer)r4r,rr5r7r8r+s    z&MistralForTokenClassification.__init__cCrr9rrr7r7r8rrz2MistralForTokenClassification.get_input_embeddingscCrr9rrr7r7r8rrz2MistralForTokenClassification.set_input_embeddingsNrr_rNrrrrrrrRc  Csd|j|||||||| d} | j} || } || } d} |dur(|| ||j} t| | | j| jdS) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). r_rNrrrrrN)rrrPr) rrrar rr,rrPr)r4rr_rNrrrrrrrsequence_outputrrr7r7r8r;s,  z%MistralForTokenClassification.forwardr)r<r=r>r+rrrrrrErrr rrrr;r?r7r7r5r8rsH    ra The Mistral Model transformer with a sequence classification head on top (linear layer). [`MistralForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). )Z custom_introcr) MistralForSequenceClassificationcs@t||j|_t||_tj|j|jdd|_| dSr') r*r+r rrrr.r-r rr3r5r7r8r+'s   z)MistralForSequenceClassification.__init__cCrr9rrr7r7r8r0rz5MistralForSequenceClassification.get_input_embeddingscCrr9rrr7r7r8r3rz5MistralForSequenceClassification.set_input_embeddingsNrr_rNrrrrrrrRc  Cs(|j|||||||| d} | j} || } |dur|jd} n|jd} |jjdur2| dkr2td|jjdur;d}n1|dur`||jjk| jt j }t j |jd| jt j d}|| d}n d}t |jjd| t j | | jd |f}d}|dur|j| |||jd }t||| j| j| jd S) r r Nrr"z=Cannot handle batch sizes > 1 if no padding token is defined.r@)rrcz will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r)rr pooled_logitsr,r)rrr rDr,rrrjrrEZint32rZargmaxrrr6r<rrrrPr)r4rr_rNrrrrrrZtransformer_outputsrPrrZlast_non_pad_tokenZ non_pad_maskZ token_indicesrrr7r7r8r;6sL     z(MistralForSequenceClassification.forwardr)r<r=r>r+rrrrrrErrr rrrr;r?r7r7r5r8rsH     rcseZdZdZfddZddZddZee         dd e e j d e e j d e e j d e e eee jfd e e jde e j de e j de ede edefddZZS)MistralForQuestionAnsweringrcs2t|t|jd|_t||_|dS)NrA) r*r+rr.r- qa_outputsrrrr3r5r7r8r+s   z$MistralForQuestionAnswering.__init__cCrr9rrr7r7r8rrz0MistralForQuestionAnswering.get_input_embeddingscCrr9rrr7r7r8rrz0MistralForQuestionAnswering.set_input_embeddingsNrr_rNrrstart_positions end_positionsrrrRc  Ks|j||||||| d} | j} || } | jddd\}}|d}|d}d}|durA|durA|j||||fi| }t|||| j| j dS)a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. )r_rNrrrrr"r@rBN)r start_logits end_logitsrPr) rrrsplitZsqueezerkrrrPr)r4rr_rNrrrrrrrlrr rrrrr7r7r8r;s0 z#MistralForQuestionAnswering.forwardr)r<r=r>rr+rrrrrrErrrr rrrrr;r?r7r7r5r8r|sJ    r)rrrrrr)Nr")rZ)KtypingrrrrrrErZ activationsr Z cache_utilsr r r r Z generationrZ integrationsrZmodeling_attn_mask_utilsrZmodeling_flash_attention_utilsrZmodeling_layersrZmodeling_outputsrrrrrZmodeling_rope_utilsrrZmodeling_utilsrrZprocessing_utilsrutilsrrrr r!Zconfiguration_mistralr#Z!torch.nn.attention.flex_attentionr$Zintegrations.flex_attentionr%Z get_loggerr<rModuler&rHrOrrrYrrrrsrrrrrrrrrr__all__r7r7r7r8s~              D3"lFV I