o Zh8m@sddlmZddlmZmZmZmZddlZddlmZddl m Z ddl m Z ddl mZdd lmZdd lmZmZdd lmZdd lmZdd lmZmZmZmZddlmZddlm Z edGdddej!Z"Gdddej!Z#Gdddej!Z$eGdddeZ%eGdddeZ&eGdddeZ'edd Gd!d"d"e'Z(Gd#d$d$eeZ)ed%d Gd&d'd'e'e Z*gd(Z+dS))) dataclass)ListOptionalTupleUnionN)nn)ACT2FN)GenerationMixin)use_kernel_forward_from_hub)FlashAttentionKwargs)BaseModelOutputWithPast ModelOutput)PreTrainedModel)Unpack) LossKwargsauto_docstringcan_return_tupleis_torchdynamo_compiling) AutoModel)Mistral3ConfigZRMSNormcs.eZdZdfdd ZddZddZZS) Mistral3RMSNormư>cs&ttt||_||_dS)z> Mistral3RMSNorm is equivalent to T5LayerNorm N)super__init__r ParametertorchZonesweightvariance_epsilon)self hidden_sizeeps __class__]/var/www/auris/lib/python3.10/site-packages/transformers/models/mistral3/modeling_mistral3.pyr/s  zMistral3RMSNorm.__init__cCsJ|j}|tj}|djddd}|t||j}|j||S)NrT)Zkeepdim) dtypetorZfloat32powmeanZrsqrtr r)r! hidden_statesZ input_dtypeZvariancer&r&r'forward7s  zMistral3RMSNorm.forwardcCst|jjd|jS)Nz, eps=)tuplershaper r!r&r&r' extra_repr>szMistral3RMSNorm.extra_repr)r)__name__ __module__ __qualname__rr.r2 __classcell__r&r&r$r'r-srcsBeZdZdZdeffdd Zdejdejdejfdd ZZ S) Mistral3PatchMergerz< Learned merging of spatial_merge_size ** 2 patches configcsLt||_|jj}|j|_|jjj|_tj||jd|dd|_ dS)NrFbias) rrr8 vision_configr"spatial_merge_size patch_sizerLinear merging_layer)r!r8r"r$r&r'rGs   zMistral3PatchMerger.__init__image_features image_sizesreturnc sfdd|D}dd|D}|jd}g}t||D]7\}}||\}} ||| |dddd} tjjj | j j d} | |j dd } | | qtj |dd }|}|S) Ncs(g|]}|dj|djfqS)rr)r=).0Z image_sizer1r&r' Qsz/Mistral3PatchMerger.forward..cSsg|]\}}||qSr&r&)rChwr&r&r'rDUsr(rrr)Z kernel_sizeZstridedim)r0 enumeratesplitviewZpermute unsqueezerrZ functionalZunfoldr<tappendcatr?) r!r@rAZtokens_per_imagedZpermuted_tensorZ image_indexZ image_tokensrErFZ image_gridgridr&r1r'r.Ps"      zMistral3PatchMerger.forward) r3r4r5__doc__rrrTensorr.r6r&r&r$r'r7Bs$ r7cs8eZdZdeffdd ZdejdejfddZZS)Mistral3MultiModalProjectorr8cstt|jj|jjd|_t||_ t |j t rdnt |j }tj|jj||jj|jd|_t|j|_tj|jj|jj|jd|_dS)N)r#rr9)rrrr;r" text_configZ rms_norm_epsnormr7 patch_merger isinstancevision_feature_layerintlenrr>Zmultimodal_projector_biaslinear_1r Zprojector_hidden_actactlinear_2)r!r8Znum_feature_layersr$r&r'ris     z$Mistral3MultiModalProjector.__init__r@rAcCs8||}|||}||}||}||}|SN)rVrWr\r]r^)r!r@rAr-r&r&r'r.ys     z#Mistral3MultiModalProjector.forward) r3r4r5rrrrSr.r6r&r&r$r'rThsrTc@seZdZUdZdZeejed<dZ eejed<dZ ee ejed<dZ ee ejed<dZee ejed<dZeejed<dS) Mistral3CausalLMOutputWithPasta Base class for Mistral3 causal language model (or autoregressive) outputs. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. image_hidden_states (`torch.FloatTensor`, *optional*): A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`. image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state. Nlosslogitspast_key_valuesr- attentionsimage_hidden_states)r3r4r5rRrarr FloatTensor__annotations__rbrcrr-rrdrer&r&r&r'r`s r`c@s$eZdZUdZdZeejed<dS)Mistral3ModelOutputWithPasta Base class for Mistral3 outputs, with hidden states and attentions. Args: last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. image_hidden_states (`torch.FloatTensor`, *optional*): A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`. image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state. Nre) r3r4r5rRrerrrfrgr&r&r&r'rhs rhc@s<eZdZeZdZdZdZdZdZ dZ dZ dZ dZ ddZdS)Mistral3PreTrainedModelTrccCst|jd|jj}t|tjr)|jjj d|d|j dur'|j j dSdSt|tj r>|jj d|j j dSt|trL|jj ddSdS)Ninitializer_rangeg)r,stdg?)getattrr8Zget_text_configrkrXrr>rdataZnormal_r:Zzero_Z LayerNormZfill_r)r!modulerlr&r&r' _init_weightss    z%Mistral3PreTrainedModel._init_weightsN)r3r4r5rZ config_classZbase_model_prefixZsupports_gradient_checkpointingZ_skip_keys_device_placementZ_supports_cache_classZ_supports_flash_attn_2Z_supports_sdpaZ_supports_quantized_cacheZ_supports_static_cacheZ_supports_attention_backendrpr&r&r&r'ris rizx The Mistral3 model which consists of a vision backbone and a language model, without a language modeling head. )Z custom_introc#s&eZdZddiZdeffdd ZddZdd Z dd ej d ej d e e e ee ffddZee ddejd ej de ej de ejde eej de ej d e e e ee fde ede ede ede ede ejd ej deede eeffddZZS) Mistral3Modelzlanguage_model.modellanguage_modelr8cs>t|t|j|_t||_t|j|_ | dSr_) rrrZ from_configr; vision_towerrTmulti_modal_projectorrUrr post_initr!r8r$r&r'rs   zMistral3Model.__init__cC |jSr_)rrget_input_embeddingsr1r&r&r'rx z"Mistral3Model.get_input_embeddingscC|j|dSr_)rrset_input_embeddingsr!valuer&r&r'r{z"Mistral3Model.set_input_embeddingsN pixel_valuesrArYc s|dur|n|jj}dd|D}|j|f|dd|t|tr*j|}nfdd|D}tj|dd }| | d |}|S) aU Obtains image last hidden states from the vision tower and apply multimodal projection. Args: pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`): The tensors corresponding to the input images. vision_feature_layer (`Union[int, List[int]]`, *optional*): The index of the layer to select the vision feature. If multiple indices are provided, the vision feature of the corresponding indices will be concatenated to form the vision features. image_sizes (`torch.Tensor`, *optional*): Tensor containing the image sizes as returned by the processor. Returns: image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`). NcSsi|] \}}|dur||qSr_r&)rCkvr&r&r' sz4Mistral3Model.get_image_features..T)rAoutput_hidden_statescsg|]}j|qSr&)r-)rCZ layer_idxZ image_outputsr&r'rD!sz4Mistral3Model.get_image_features..r(rGr) r8rYitemsrsrXrZr-rrOrtZsqueeze)r!rrArYkwargsZselected_image_featureZhs_poolr@r&rr'get_image_featuress  z Mistral3Model.get_image_features input_idsattention_mask position_idsrc inputs_embeds use_cacheoutput_attentionsr return_dictcache_positionrrBc Ks| dur| n|jj} | dur| n|jj} | dur| n|jj} |dur$|n|jj}|du|duAr4td|dur@|dur@td|durJ||}|dur|j||| d}||jjk d}| | |j }t s|||kr||jjk}|jd|jd}td|d|| |j |j}|||}|jd |||||| | d | d |}t|j|j|j|j|dur|d Sdd S) Nz:You must specify exactly one of input_ids or inputs_embedszdYou cannot specify both pixel_values and inputs_embeds at the same time, and must specify either one)rrYrAr(rrz6Image features and image tokens do not match: tokens: z , features T) rrrcrrrrrr)last_hidden_statercr-rdrer&)r8rruse_return_dictrY ValueErrorrxrZimage_token_idrLZ expand_asr*devicerZnumelsumr0r)Zmasked_scatterrrrhrrcr-rd)r!rrrrrcrrYrrrrrrArr@Zspecial_image_maskZn_image_tokensZn_image_featuresoutputsr&r&r'r.'sh     zMistral3Model.forwardr_) NNNNNNNNNNNNN)r3r4r5_checkpoint_conversion_mappingrrrxr{rrfrSrrrZrrrr LongTensorboolrr rrhr.r6r&r&r$r'rqsv (       rqc@s eZdZdS)KwargsForCausalLMN)r3r4r5r&r&r&r'rvs rzV The MISTRAL3 model which consists of a vision backbone and a language model. c%seZdZdddddZdgZdeffdd Zd d Zd d Zde j fddZ ddZ e ddZe ddZe ddZee              d4dejdejdeejdeejdeeejd eejd!eejd"eed#eed$eed%eed&eejd'eeejfd(eejd)eedeeeff d*d+Z      d5fd,d- Z!e"dejd.ed/ed0ej#d&ejd1ef d2d3Z$Z%S)6 Mistral3ForConditionalGenerationzmodel.language_modelzmodel.vision_towerzmodel.multi_modal_projectorlm_head)z^language_model.modelz ^vision_towerz^multi_modal_projectorz^language_model.lm_headzlm_head.weightr8cs<t|t||_tj|jj|jjdd|_ | dS)NFr9) rrrqmodelrr>rUr" vocab_sizerrurvr$r&r'rs   z)Mistral3ForConditionalGeneration.__init__cCrwr_)rrxr1r&r&r'rxryz5Mistral3ForConditionalGeneration.get_input_embeddingscCrzr_)rr{r|r&r&r'r{r~z5Mistral3ForConditionalGeneration.set_input_embeddingsrBcCs|jSr_rr1r&r&r'get_output_embeddingssz6Mistral3ForConditionalGeneration.get_output_embeddingscCs ||_dSr_r)r!Znew_embeddingsr&r&r'set_output_embeddingsryz6Mistral3ForConditionalGeneration.set_output_embeddingscC|jjSr_)rrrr1r&r&r'rrz/Mistral3ForConditionalGeneration.language_modelcCrr_)rrsr1r&r&r'rsrz-Mistral3ForConditionalGeneration.vision_towercCrr_)rrtr1r&r&r'rtrz6Mistral3ForConditionalGeneration.multi_modal_projectorNrrrrrrcrlabelsrrrrrlogits_to_keeprArcKs| dur| n|jj} | dur| n|jj} | dur| n|jj} |jd|||||||| | d| |d |}|d}t| trBt| dn| }||dd|ddf}d}|durg|j d|||jj j d|}t |||j |j|j|jdS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, Mistral3ForConditionalGeneration >>> model = Mistral3ForConditionalGeneration.from_pretrained("mistralai/Mistral-Small-3.1-24B-Instruct-2503") >>> processor = AutoProcessor.from_pretrained("mistralai/Mistral-Small-3.1-24B-Instruct-2503") >>> prompt = "[INST][IMG]What is the image?[/INST]" >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, text=prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(**inputs, max_new_tokens=15) >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "What is the image?The image depicts two cats lying on a pink blanket." ```NT) rrrrrcrrrrrrrAr)rbrr)rarbrcr-rdrer&)r8rrrrrXrZslicerZ loss_functionrUrr`rcr-rdre)r!rrrrrcrrrrrrrrrArrr-Z slice_indicesrbrar&r&r'r.sL/  z(Mistral3ForConditionalGeneration.forwardc  s8tj|f|||||d|} |ddkr|| d<| S)N)rcrrrrrr)rprepare_inputs_for_generation) r!rrcrrrrrrZ model_inputsr$r&r'rs  z>Mistral3ForConditionalGeneration.prepare_inputs_for_generationsequence_length target_lengthr) batch_sizec KsD|dur|dkr|}|St|j}tj||f|||jd}|dkr+tj|dd}|tj||jd|ddk9}|ddddddf |ddd}|dur| }|j d} |ddddddd| f|ddddddf |j} | dk} |ddddddd| f | ||ddddddd| f<|S) aM Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. N)Z fill_valuer)rr)Zdiagonal)rr(r)rHrZfinfominfullrZtriuZarangeZreshapeexpandcloner0r*Z masked_fill) rrrr)rrrZ causal_maskZ min_dtypeZ mask_lengthZ padding_maskr&r&r'5_prepare_4d_causal_attention_mask_with_cache_positions,  $ 6  zVMistral3ForConditionalGeneration._prepare_4d_causal_attention_mask_with_cache_position)NNNNNNNNNNNNrN)NNNNNN)&r3r4r5rZ_tied_weights_keysrrrxr{rModulerrpropertyrrrsrtrrrrrfrrSrrrrZrrrr`r.r staticmethodr)rr6r&r&r$r'rys           Zr)rqrir), dataclassesrtypingrrrrrrZ activationsr Z generationr Z integrationsr Zmodeling_flash_attention_utilsr Zmodeling_outputsr rZmodeling_utilsrZprocessing_utilsrutilsrrrrautorZconfiguration_mistral3rrrr7rTr`rhrirqrr__all__r&r&r&r'sH          &'   X