o Zhn@sddlmZmZmZmZddlZddlmZddlm Z ddl m Z ddl m Z mZddlmZmZddlmZdd lmZmZmZmZd d lmZeeZGd d d ejZGdddejZ GdddejZ! d7dejdej"dej"dej"deej"de#de#fddZ$ddZ%dej"de&d ej"fd!d"Z'd#ej"d$ej"d%ej"d&ej"d eej"ej"ff d'd(Z(Gd)d*d*ejZ)Gd+d,d,ejZ*Gd-d.d.ejZ+Gd/d0d0ejZ,eGd1d2d2eZ-ed3d4Gd5d6d6e-Z.d2d6gZ/dS)8)CallableOptionalTupleUnionN)ACT2FN)FlashAttentionKwargs)BaseModelOutputBaseModelOutputWithPooling)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstringcan_return_tuplelogging torch_int)MLCDVisionConfigcs2eZdZfddZdejdejfddZZS)MLCDMLPcsDt||_t|j|_t|j|j |_ t|j |j|_ dSN) super__init__configrZ hidden_act activation_fnnnLinear hidden_sizeZintermediate_sizefc1fc2selfr __class__U/var/www/auris/lib/python3.10/site-packages/transformers/models/mlcd/modeling_mlcd.pyr's  zMLCDMLP.__init__ hidden_statesreturncCs"||}||}||}|Sr)rrr)r r%r#r#r$forward.s   zMLCDMLP.forward)__name__ __module__ __qualname__rtorchTensorr' __classcell__r#r#r!r$r&s rcsDeZdZd dededdffdd Zded edejfd d ZZ S) MLCDRotaryEmbedding@dimthetar&Ncs>td|tjd|dtjd|}|jd|dddS)N?rdtypeinv_freqF persistent)rrr+arangefloatregister_buffer)r r0r1r6r!r#r$r6s  zMLCDRotaryEmbedding.__init__num_patches_heightnum_patches_widthc Cstj||jjddd|}tj||jjdd|d}tj||gdd}t||}tj||jj|jj d}t ||j}||d} | S)a} Calculate the Rotary Position Embedding (RoPE) for MLCDVisionModel based on the grid size. Args: num_patches_height (int): Number of patches in the height dimension. num_patches_width (int): Number of patches in the width dimension. Returns: torch.Tensor: Rotary positional embeddings for the given grid size. )devicerrr0)r>r5) r+r9r6r> unsqueezeexpandstackflattenmaxr5outer) r r<r=Zhpos_idsZwpos_idsZpos_idsZ max_grid_sizeseqZrotary_pos_emb_fullrotary_pos_embr#r#r$r';s  zMLCDRotaryEmbedding.forward)r/) r(r)r*intr:rr+r,r'r-r#r#r!r$r.5s r.csVeZdZdeffdd Zdejdededejfdd Zd ej dejfd d Z Z S) MLCDVisionEmbeddingsrcst||_|j|_|j|_|j|_tt |j|_ tj |j |j|j|jdd|_|j|jd|_|jd|_|jdt |jddddS)NF)Z in_channelsZ out_channelsZ kernel_sizeZstridebiasr3r position_ids)rr?r7)rrrr embed_dimZ image_size patch_sizer Parameterr+randnclass_embeddingZConv2dZ num_channelspatch_embedding num_patches num_positionsr;r9rBrr!r#r$r]s  "zMLCDVisionEmbeddings.__init__ embeddingsheightwidthr&c Cs|jdd}|jjd}|jdd}tjs(||kr(||kr(||jS|ddddf}|ddddf}|jd} ||j} ||j} t |d} | d| | | }| dddd}t j j|| | fdd d }| dddddd| }tj||fdd S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 rrNr?g?rr3ZbicubicF)sizemodeZ align_cornersr@)shapeposition_embeddingweightrAr+Zjit is_tracingrLrNrreshapepermuter functionalZ interpolateviewcat) r rUrVrWrSr[rTZclass_pos_embedZpatch_pos_embedr0Z new_heightZ new_widthZsqrt_num_positionsr#r#r$interpolate_pos_encodingrs*      z-MLCDVisionEmbeddings.interpolate_pos_encoding pixel_valuescCs^|jd}|jjj}||j|d}|ddd}|j|dd}t j ||gdd}|S)Nrr4r3rr?r@) rZrRr\r5torD transposerQrBr+rb)r rd batch_sizeZ target_dtypeZ patch_embedsZ class_embedsrUr#r#r$r's  zMLCDVisionEmbeddings.forward) r(r)r*rrr+r,rIrc FloatTensorr'r-r#r#r!r$rJ\s)rJmodulequerykeyvalueattention_maskscalingdropoutc Kst||j}t||j} t||dd|} |dur3|ddddddd|jdf} | | } tjj| dtj d |j } tjj | ||j d} t| | } | dd} | | fS)Nr3rr?)r0r5)ptrainingr) repeat_kvnum_key_value_groupsr+matmulrfrZrr`ZsoftmaxZfloat32rer5rprs contiguous) rjrkrlrmrnrorpkwargs key_states value_states attn_weightsZ causal_mask attn_outputr#r#r$eager_attention_forwards & r}cCsH|dd|jddf}|d|jdddf}tj| |fddS)z*Rotates half the hidden dims of the input..Nr?r3r@)rZr+rb)xx1Zx2r#r#r$ rotate_halfsrr%n_repr&cCs^|j\}}}}|dkr |S|dddddddddf|||||}||||||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)rZrBr^)r%rbatchZnum_key_value_headsslenhead_dimr#r#r$rts 0rtqkcossincCs|j}|j}||}}|d|d}}||t||}||t||}||}||}||fS)Nrq)r5r:rArre)rrrrZ orig_q_dtypeZ orig_k_dtypeZq_embedZk_embedr#r#r$apply_rotary_pos_emb_visions  rcspeZdZdZdeffdd Z d dejdeejejfde ejd e e d eeje ejff d d Z Z S) MLCDAttentionaAMulti-headed attention from 'Attention Is All You Need' paper Multi-headed attention with RoPE. Refer to papers: - Attention is all you need: https://arxiv.org/abs/1706.03762 - RoFormer: Enhanced Transformer with Rotary Position Embedding: https://arxiv.org/abs/2104.09864 rcst||_|j|_|j|_|j|j|_|j|j|jkr-td|jd|jd|jd|_ |j |_ d|_ t |j|j|_t |j|j|_t |j|j|_t |j|j|_|j|_dS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).F)rrrrrMnum_attention_heads num_headsr ValueErrorscaleZattention_dropoutrp is_causalrrk_projv_projq_projout_projrurr!r#r$rs&    zMLCDAttention.__init__Nr%position_embeddingsrnrxr&cKs|jdd\}}|||||j|jf}|||||j|jf}|||||j|jf} |dd} |dd} t ||| | \}}| dddd }| dddd }| dddd } t } |j jdkr|j jdkr|d d rtd nt|j j} | |||| |f|jsd n|j|j|jd |\} }| dddd } | ||d} || } | ddd } | |fS)z#Input shape: Batch x Time x ChannelNr?rrr3reagerZsdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.ri)rpror)rZrr^rrrrrAr:rr_rwr}rZ_attn_implementationgetloggerZ warning_oncer rsrprrrar)r r%rrnrxrgZ seq_lengthZ query_statesryrzrrZattention_interfacer|r{r#r#r$r'sF     zMLCDAttention.forwardr)r(r)r*__doc__rrr+r,rrr rr'r-r#r#r!r$rsrc sdeZdZdeffdd Z  d dejdeejejfdeejd ee d eej f d d Z Z S)MLCDEncoderLayerrcsRt|j|_t||_tj|j|jd|_ t ||_ tj|j|jd|_ dS)NZeps) rrrrMr self_attnr LayerNormlayer_norm_eps layer_norm1rmlp layer_norm2rr!r#r$r8s   zMLCDEncoderLayer.__init__NFr%rrnrr&cCsd|}||}|j||||d\}}||}|}||}||}||}|f}|r0||f7}|S)a Args: hidden_states (`torch.FloatTensor`): Input to the layer of shape `(batch, seq_len, embed_dim)`. Represents the hidden states from the previous layer or the input embeddings. position_embeddings (`Tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.FloatTensor`): Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values. output_attentions (`bool`, *optional*, defaults to `False`): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. r%rrnr)rrrr)r r%rrnrZresidualr{Zoutputsr#r#r$r'@s"     zMLCDEncoderLayer.forward)NF) r(r)r*rrr+r,rrboolrhr'r-r#r#r!r$r7s rcseZdZdZdeffdd Ze    ddejde ej ej fde ej d e e d e e d e e d e e effd dZZS) MLCDEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`MLCDEncoderLayer`]. Args: config: MLCDVisionConfig rcs:t|_tfddtjD|_d|_dS)z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.csg|]}tqSr#)r).0_rr#r$ zsz(MLCDEncoder.__init__..FN) rrrrZ ModuleListrangenum_hidden_layerslayersgradient_checkpointingrr!rr$rvs   zMLCDEncoder.__init__N inputs_embedsrrnroutput_hidden_states return_dictr&c Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}|r"dnd}|r(dnd}|} t|jD]1\} } |r<|| f}|jrM|jrM|| j | |||} n| | |||d} | d} |rb|| df}q1|rj|| f}|sxt dd| ||fDSt | ||dS) aj Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. position_embeddings (`Tuple[torch.Tensor, torch.Tensor]`): A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`. Represents absolute positional embeddings for the query and key in the attention mechanism. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr#rrrcss|] }|dur|VqdSrr#)rvr#r#r$ sz&MLCDEncoder.forward..)last_hidden_stater% attentions) rruse_return_dictr enumeraterrrsZ_gradient_checkpointing_func__call__tupler ) r rrrnrrrZencoder_statesZall_attentionsr%idxZ encoder_layerZ layer_outputsr#r#r$r'}sJ#     zMLCDEncoder.forwardNNNN)r(r)r*rrrrr+rhrr,rrrr r'r-r#r#r!r$rms. rcsheZdZdeffdd Ze    d deejdee dee dee d e e e ff d d Z ZS) MLCDVisionTransformerrcst||_|j}t||_tj||jd|_ t ||_ tj||jd|_ t |j|jd|_ttd|j|jd|_dS)Nrr3r)rrrrrJrUrrr pre_layrnormrencoderpost_layernormr.rvision_rotary_embeddingrOr+rP class_pos_emb)r rrMr!r#r$rs   $zMLCDVisionTransformer.__init__Nrdrrrr&cCs<|dur|n|jj}|dur|n|jj}|dur|n|jj}|dur&td|jd|jj}|jd|jj}|||}||j j }t j |j |gdd}t j ||fdd}| |f} ||} || } |j| | |||d} | d} | dddddf} || } |s| | f| ddSt| | | j| jdS) Nz You have to specify pixel_valuesrqr?rr@)rrrrrr)rZ pooler_outputr%r)rrrrrrZrNrrerr>r+rbrrrUrrrr r%r)r rdrrrr<r=rHZembrr%Zencoder_outputsrZ pooled_outputr#r#r$r'sB     zMLCDVisionTransformer.forwardr)r(r)r*rrrrr+rhrrrr r'r-r#r#r!r$rs$  rc@s(eZdZeZdZdZdZdZddZ dS)MLCDPreTrainedModelZmlcdTcCs|jj}t|tr,|jj}tjj|jd|jd|dtjj|j j |jj |ddSt|t ru|jj}|jdd|jj d|}|jd|}tjj|jj |dtjj|jj |dtjj|jj |dtjj|jj |ddSt|tr|jj}|jjdd|jj d|}d|jjd|}tjj|jj |dtjj|jj |ddSt|tr|jj}|jj|jjdd|}tjj|jd|ddSt|tjr|jj|j jddSt|tjr|jdur|jjdSdSdS)zInitialize the weightsrir)meanstd)rr3r2N)rZinitializer_factor isinstancerJrinitZnormal_rQrMrRr\Zinitializer_rangerrrrrrrrrrrrrrrKdataZzero_Zfill_r)r rjfactorZ in_proj_stdZ out_proj_stdZfc_stdZ pos_emb_stdr#r#r$ _init_weightss:        z!MLCDPreTrainedModel._init_weightsN) r(r)r*r config_classZbase_model_prefixZsupports_gradient_checkpointingZ_supports_flash_attn_2Z_supports_sdparr#r#r#r$r s rzN The vision model from M_L_C_D without any head or projection on top. )Z custom_introcseZdZeZdZdgZdeffdd Zdej fddZ e dde e jd e ed e ed e edeeeff d dZZS)MLCDVisionModelrdrrcs"t|t||_|dSr)rrr vision_modelZ post_initrr!r#r$r?s   zMLCDVisionModel.__init__r&cCs |jjjSr)rrUrR)r r#r#r$get_input_embeddingsEs z$MLCDVisionModel.get_input_embeddingsNrrrcCsN|dur|n|jj}|dur|n|jj}|dur|n|jj}|j||||dS)a Example: ```python >>> import requests >>> from PIL import Image >>> from transformers import AutoProcessor, MLCDVisionModel >>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs, output_attentions=True) >>> features = outputs.last_hidden_state >>> print(f"Extracted features shape: {features.shape}") >>> print(f"Number of attention layers: {len(outputs.attentions)}") >>> print(f"Attention shape: {outputs.attentions[0].shape}") ```N)rdrrr)rrrrr)r rdrrrr#r#r$r'HszMLCDVisionModel.forwardr)r(r)r*rrZmain_input_nameZ_no_split_modulesrrModulerrrr+rhrrrr r'r-r#r#r!r$r5s, r)ri)0typingrrrrr+Ztorch.nnrZ activationsrZmodeling_flash_attention_utilsrZmodeling_outputsr r Zmodeling_utilsr r Zprocessing_utilsr utilsrrrrZconfiguration_mlcdrZ get_loggerr(rrrr.rJr,r:r}rrIrtrrrrrrr__all__r#r#r#r$sl      'S   T6`@' :