o Zh.A@sddlmZmZmZmZmZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZddlmZmZdd lmZdd lmZdd lmZmZd d lmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'm(Z(er}ddl)m*Z*ddl+m,Z,e-e.Z/dZ0Gddde"Z1GdddeZ2GdddeZ3Gddde$Z4Gddde#Z5GdddeZ6Gdd d e!Z7Gd!d"d"e Z8Gd#d$d$eZ9gd%Z:dS)&)CallableListOptionalTupleUnionN)nn)CacheSlidingWindowCache StaticCache)AttentionMaskConverter)FlashAttentionKwargs)BaseModelOutputWithPastQuestionAnsweringModelOutput)ALL_ATTENTION_FUNCTIONS)Unpack)is_torch_flex_attn_availablelogging) LlamaAttentionLlamaDecoderLayerLlamaForCausalLMLlamaForQuestionAnsweringLlamaForSequenceClassificationLlamaForTokenClassificationLlamaMLP LlamaModelLlamaPreTrainedModelapply_rotary_pos_embeager_attention_forward) MistralConfig) BlockMask)make_flex_block_causal_maskzmistralai/Mistral-7B-v0.1cseZdZfddZZS) MistralMLPcsRt|tj|j|jdd|_tj|j|jdd|_tj|j|jdd|_dS)NFZbias) super__init__rLinear hidden_sizeZintermediate_sizeZ gate_projZup_projZ down_projselfconfig __class__Z/var/www/auris/lib/python3.10/site-packages/transformers/models/mistral/modular_mistral.pyr'*s zMistralMLP.__init__)__name__ __module__ __qualname__r' __classcell__r/r/r-r0r$)sr$cseZdZdedeffdd Z  ddejdeejejfde ejd e e d e ej d e e d eeje eje eejffd dZZS)MistralAttentionr, layer_idxcstt|ddp|j|j|_tj|j|j|jdd|_tj|j|j |jdd|_ tj|j|j |jdd|_ tj|j|j|jdd|_ dS)Nhead_dimFr%) r&r'getattrr)Znum_attention_headsr7rr(q_projZnum_key_value_headsk_projv_projo_projr+r,r6r-r/r0r'2s  zMistralAttention.__init__N hidden_statesposition_embeddingsattention_maskpast_key_valuecache_positionkwargsreturncKsT|jdd}g|d|jR}|||dd} |||dd} |||dd} |\} } t| | | | \} } |durW| | |d}|| | |j |\} } t }|j j dkrw|j j dkrq| ddrqtd nt|j j }||| | | |f|jsd n|j|jt|j d dd |\}}|jg|dR}||}||fS) Nr r)sincosrBeagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.sliding_window)ZdropoutscalingrL)shaper7r9viewZ transposer:r;rupdater6rr,_attn_implementationgetloggerZ warning_oncertrainingZattention_dropoutrMr8reshape contiguousr<)r+r>r?r@rArBrCZ input_shapeZ hidden_shapeZ query_statesZ key_statesZ value_statesrGrFZ cache_kwargsZattention_interfaceZ attn_outputZ attn_weightsr/r/r0forward:sB       zMistralAttention.forward)NN)r1r2r3r!intr'torchTensorrrr LongTensorrr rWr4r/r/r-r0r51s& r5cs&eZdZdedeffdd ZZS)MistralDecoderLayerr,r6cs*t||t||d|_t||_dS)N)r,r6)r&r'r5Z self_attnr$Zmlpr=r-r/r0r'nszMistralDecoderLayer.__init__)r1r2r3r!rXr'r4r/r/r-r0r\msr\c@ eZdZdS)MistralPreTrainedModelNr1r2r3r/r/r/r0r^tr^cseZdZdeffdd Z ddeejdfdejdejd ed e f d d Z e dejd e de dej dejde ded efddZZS) MistralModelr,cs0ttfddtjD|_dS)Ncsg|]}t|qSr/)r\).0r6r,r/r0 |sz)MistralModel.__init__..)r&r'rZ ModuleListrangeZnum_hidden_layersZlayersr*r-rcr0r'ys  zMistralModel.__init__Fr@r" input_tensorrBpast_key_valuesrJc Cs|jjdkr2|dur&|dur&|dddf|dk}|r&td|dur0d|vr0|SdS|jjdkrDt|tjrBt |}|S|durL| nd}t|t }t|t } |jjdkrs|ss| ss|sst j||||jj|jdrsdS|j} t| j} |jd } | s|r|} nt|tjr|jdn|| d } |j|| | | ||jd|j|d }|jjdkr|dur|jjd vr|st || }|S) NZflash_attention_2rErzYou are attempting to perform batched generation with padding_side='right' this may lead to unexpected behaviour for Flash Attention version of Mistral. Make sure to call `tokenizer.padding_side = 'left'` before tokenizing the input. rKZflex_attentionrI) inputs_embedsZpast_key_values_lengthrLZ is_trainingr )sequence_length target_lengthdtyperB batch_sizer,rg)cudaZxpuZnpu)r,rQsumitemsize ValueError isinstancerYrZr#Zget_seq_lengthr r r Z_ignore_causal_mask_sdparLrTrkfinfominrNZget_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positiondevicetypeZ_unmask_unattended)r+r@rfrBrgrJZis_padding_rightZpast_seen_tokensZusing_static_cacheZusing_sliding_window_cacherk min_dtyperirj causal_maskr/r/r0_update_causal_maskst $             z MistralModel._update_causal_maskrirjrkrlcCs|dur|dkr|}|St|j} tj||f| ||jd}tj||jd|ddk} |} t | ddr\| j dur\t |t rF||kr\tj||jd|dd| j k} | | || 9}|ddddddf|ddd}|dur|}|jd|kr|ddd|f}|jd} |ddddddd| f|ddddddf|j}|d k}|ddddddd| f|| |ddddddd| f<|S) a Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. config (`MistralConfig`): The model's configuration class past_key_values (`Cache`): The cache class that is being used currently to generate N)Z fill_valuerkrv)rvrEr Zuse_sliding_windowTr)dimrYrsrtfullrvZarangerUZget_text_configr8rLrrr Z bitwise_or_expandclonerNtoZ masked_fill)r@rirjrkrBrlr,rgryrxZdiagonal_attend_maskZ text_configZsliding_attend_maskZ mask_lengthZ padding_maskr/r/r0rus@!  $ 6  zBMistralModel._prepare_4d_causal_attention_mask_with_cache_position)F)r1r2r3r!r'rrYrZr boolrz staticmethodrXrkrur4r/r/r-r0raxs@   Vrac@r])MistralForCausalLMNr_r/r/r/r0rr`rc@r])MistralForTokenClassificationNr_r/r/r/r0rr`rc@r]) MistralForSequenceClassificationNr_r/r/r/r0r#r`rcseZdZdZfddZddZddZ         dd eej d eej d eej d ee e e ejfd eejdeej deej deedeedefddZZS)MistralForQuestionAnsweringmodelcst|t||_|`dSN)r&r'rarZ transformerr*r-r/r0r'*s  z$MistralForQuestionAnswering.__init__cCs|jjSrrZ embed_tokens)r+r/r/r0get_input_embeddings/sz0MistralForQuestionAnswering.get_input_embeddingscCs ||j_dSrr)r+valuer/r/r0set_input_embeddings2s z0MistralForQuestionAnswering.set_input_embeddingsN input_idsr@ position_idsrgrhstart_positions end_positionsrJoutput_hidden_statesrDc  Ks|j||||||| d} | j} || } | jddd\}}|d}|d}d}|durA|durA|j||||fi| }t|||| j| j dS)a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. )r@rrgrhrJrr rE)r|N)loss start_logits end_logitsr> attentions) rZlast_hidden_stateZ qa_outputssplitZsqueezerVZ loss_functionrr>r)r+rr@rrgrhrrrJrrCZoutputsZsequence_outputZlogitsrrrr/r/r0rW5s0 z#MistralForQuestionAnswering.forward) NNNNNNNNN)r1r2r3Zbase_model_prefixr'rrrrYr[rZrr rZ FloatTensorrrrWr4r/r/r-r0r'sF    r)rrrar^rr);typingrrrrrrYZtorch.utils.checkpointrZ cache_utilsr r r Zmodeling_attn_mask_utilsr Zmodeling_flash_attention_utilsr Zmodeling_outputsrrZmodeling_utilsrZprocessing_utilsrutilsrrZllama.modeling_llamarrrrrrrrrrrZconfiguration_mistralr!Z!torch.nn.attention.flex_attentionr"Zintegrations.flex_attentionr#Z get_loggerr1rSZ_CHECKPOINT_FOR_DOCr$r5r\r^rarrrr__all__r/r/r/r0s:     4    <$ D