o Zh@sdZddlZddlmZddlmZmZmZmZm Z m Z ddl Z ddl m Z ddlZ ddlmZddlmZmZmZmZmZddlmZmZdd lmZmZmZdd lmZm Z m!Z!m"Z"d d l#m$Z$m%Z%m&Z&e!'e(Z)d e j*de j*fddZ+de j*de j*fddZ,eGdddeZ-Gddde j.Z/Gddde j.Z0Gddde j.Z1de0iZ2Gddde j.Z3Gdd d e j.Z4Gd!d"d"e j.Z5Gd#d$d$e j.Z6Gd%d&d&e j.Z7Gd'd(d(e j.Z8 )dNd*e j.d+e j*d,e j*d-e j*d.ee j*d/e9d0e9fd1d2Z:Gd3d4d4e j.Z;Gd5d6d6e j.ZGd;d<dd>eZ@Gd?d@d@e j.ZAGdAdBdBe@ZBe dCdDGdEdFdFe@ZCGdGdHdHe@ZDGdIdJdJe@ZEdOdKdLZFgdMZGdS)PzPyTorch AltCLIP model.N) dataclass)AnyCallableListOptionalTupleUnion)ACT2FN)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsBaseModelOutputWithPooling,BaseModelOutputWithPoolingAndCrossAttentions'BaseModelOutputWithPoolingAndProjection)ALL_ATTENTION_FUNCTIONSPreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer) ModelOutputauto_docstringlogging torch_int) AltCLIPConfigAltCLIPTextConfigAltCLIPVisionConfiglogitsreturncCstj|tjt||jdS)Ndevice)nn functionalZ cross_entropytorcharangelenr )rr&[/var/www/auris/lib/python3.10/site-packages/transformers/models/altclip/modeling_altclip.pycontrastive_loss,sr( similaritycCs t|}t|}||dS)Ng@)r(t)r)Z caption_lossZ image_lossr&r&r' clip_loss0s  r+c@seZdZUdZdZeejed<dZ eejed<dZ eejed<dZ eejed<dZ eejed<dZ eed<dZeed <d eefd d ZdS) AltCLIPOutputa Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`): Contrastive loss for image-text similarity. logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`): The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text similarity scores. logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`): The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image similarity scores. text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPTextModel`]. image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPVisionModel`]. text_model_output (`BaseModelOutputWithPooling`): The output of the [`AltCLIPTextModel`]. vision_model_output (`BaseModelOutputWithPooling`): The output of the [`AltCLIPVisionModel`]. Nlosslogits_per_imagelogits_per_text text_embeds image_embedstext_model_outputvision_model_outputrcstfddDS)Nc3s.|]}|dvr |nt|VqdS))r2r3N)getattrto_tuple).0kselfr&r' Vs  z)AltCLIPOutput.to_tuple..)tuplekeysr8r&r8r'r5Us zAltCLIPOutput.to_tuple)__name__ __module__ __qualname____doc__r-rr# FloatTensor__annotations__r.r/r0r1r2r r3rrr5r&r&r&r'r,6s   r,cs4eZdZdZfddZ d ddZdd ZZS) AltRobertaEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. csttj|j|j|jd|_t|j|j|_ t|j |j|_ tj |j|j d|_ t|j|_t|dd|_|jdt|jddd|jd tj|jtjd dd|j|_tj|j|j|jd|_ dS) N) padding_idxZepsposition_embedding_typeabsolute position_idsrF persistenttoken_type_idsdtype)super__init__r! EmbeddingZ vocab_size hidden_sizeZ pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsZtype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutr4rFregister_bufferr#r$expandzerosrHsizelongrDr9config __class__r&r'rQcs"   zAltRobertaEmbeddings.__init__Nrc Cs|dur|durt||j|}n||}|dur|}n|dd}|d}|durTt|drI|jddd|f}||d|} | }n tj|tj |j j d}|dur]| |}| |} || } |jdkrt||} | | 7} || } || } | S)NrJrrMrrOr rG)"create_position_ids_from_input_idsrD&create_position_ids_from_inputs_embedsr`hasattrrMr^r#r_rarHr rTrWrFrVrXr\) r9 input_idsrMrH inputs_embedspast_key_values_length input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedrW embeddingsrVr&r&r'forward|s0         zAltRobertaEmbeddings.forwardcCsN|dd}|d}tj|jd||jdtj|jd}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor NrJrrfr)r`r#r$rDrar unsqueezer^)r9rkrmZsequence_lengthrHr&r&r'rhs  z;AltRobertaEmbeddings.create_position_ids_from_inputs_embeds)NNNNr)r=r>r?r@rQrrrh __classcell__r&r&rdr'rC]s   (rCcseZdZdfdd ZdejdejfddZ      dd ejd eejd eejd eejd eejdee e ejdee de ejfddZ Z S)AltRobertaSelfAttentionNcst|j|jdkrt|dstd|jd|jd|j|_t|j|j|_|j|j|_t |j|j|_ t |j|j|_ t |j|j|_ t |j|_|p\t|dd|_|jdksh|jd kry|j|_t d |jd |j|_|j|_dS) NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()rFrG relative_keyrelative_key_queryr)rPrQrSnum_attention_headsri ValueErrorintattention_head_size all_head_sizer!LinearquerykeyvaluerZZattention_probs_dropout_probr\r4rFrUrRdistance_embedding is_decoderr9rcrFrdr&r'rQs*   z AltRobertaSelfAttention.__init__xrcCs6|dd|j|jf}||}|ddddS)NrJrryrr )r`rzr}viewpermute)r9rZ new_x_shaper&r&r'transpose_for_scoress z,AltRobertaSelfAttention.transpose_for_scoresF hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionscCs||}|du} | r|dur|d} |d} |}nP| r/|||} |||} |}n;|durZ|||} |||} tj|d| gdd} tj|d| gdd} n|||} |||} ||} |du} |jrz| | f}t| | dd}|j dks|j dkr | j d| j d}}| rtj |dtj |j d dd}ntj|tj |j d dd}tj|tj |j d dd}||}|||jd}|j| jd }|j dkrtd | |}||}n|j dkr td | |}td | |}|||}|t|j}|dur||}tjj|dd}||}|dur0||}t|| }|dddd }|dd|jf}||}|rX||fn|f}|jrd||f}|S)NrrrydimrJrwrxrfrNzbhld,lrd->bhlrzbhrd,lrd->bhlrr ) rrrrr#catrmatmul transposerFshapetensorrar rr$rrUtorOZeinsummathsqrtr}r!r"softmaxr\r contiguousr`r~)r9rrrrrrrZmixed_query_layerZis_cross_attentionZ key_layerZ value_layerZ query_layer use_cacheZattention_scoresZ query_lengthZ key_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZ context_layerZnew_context_layer_shapeoutputsr&r&r'rrsn             zAltRobertaSelfAttention.forwardNNNNNNF) r=r>r?rQr#TensorrrrArboolrrrtr&r&rdr'rus4 ruc8eZdZfddZdejdejdejfddZZS)AltRobertaSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNrE) rPrQr!rrSdenserXrYrZr[r\rbrdr&r'rQ? zAltRobertaSelfOutput.__init__r input_tensorrcC&||}||}|||}|Srrr\rXr9rrr&r&r'rrE  zAltRobertaSelfOutput.forwardr=r>r?rQr#rrrrtr&r&rdr'r> $reagercseZdZdfdd ZddZ      ddejdeejd eejd eejd eejd ee e ejd ee de ejfddZ Z S)AltRobertaAttentionNcs4tt|j||d|_t||_t|_dS)NrF) rPrQ"ALT_ROBERTA_SELF_ATTENTION_CLASSES_attn_implementationr9routputset pruned_headsrrdr&r'rQSs   zAltRobertaAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrr)r%rr9rzr}rrrrrrrr~union)r9headsindexr&r&r' prune_heads[s zAltRobertaAttention.prune_headsFrrrrrrrrc Cs<||||||||}||d|} | f|dd} | S)Nrr)r9r) r9rrrrrrrZ self_outputsattention_outputrr&r&r'rrms  zAltRobertaAttention.forwardrr) r=r>r?rQrr#rrrArrrrrtr&r&rdr'rRs4 rc2eZdZfddZdejdejfddZZS)AltRobertaIntermediatecsDtt|j|j|_t|jt rt |j|_ dS|j|_ dSr) rPrQr!rrSintermediate_sizer isinstance hidden_actstrr intermediate_act_fnrbrdr&r'rQs   zAltRobertaIntermediate.__init__rrcCs||}||}|Sr)rrr9rr&r&r'rrs  zAltRobertaIntermediate.forwardrr&r&rdr'rs rcr)AltRobertaOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr) rPrQr!rrrSrrXrYrZr[r\rbrdr&r'rQrzAltRobertaOutput.__init__rrrcCrrrrr&r&r'rrrzAltRobertaOutput.forwardrr&r&rdr'rrrcseZdZfddZ      ddejdeejdeejdeejd eejd eeeejd ee d eejfd dZ ddZ Z S)AltRobertaLayercsrt|j|_d|_t||_|j|_|j|_|jr-|js&t|dt|dd|_ t ||_ t ||_ dS)Nrz> should be used as a decoder model if cross attention is addedrGr)rPrQchunk_size_feed_forward seq_len_dimr attentionradd_cross_attentionr{crossattentionr intermediaterrrbrdr&r'rQs   zAltRobertaLayer.__init__NFrrrrrrrrc Cs|dur |ddnd}|j|||||d} | d} |jr(| dd} | d} n| dd} d} |jro|durot|dsDtd|d|durN|d dnd}|| ||||||}|d} | |dd} |d} | | } t|j|j|j| }|f| } |jr| | f} | S) Nry)rrrrrJrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) rrrir{rrfeed_forward_chunkrr)r9rrrrrrrZself_attn_past_key_valueZself_attention_outputsrrZpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputs layer_outputr&r&r'rrsP         zAltRobertaLayer.forwardcCs||}|||}|Sr)rr)r9rZintermediate_outputrr&r&r'rs  z"AltRobertaLayer.feed_forward_chunkr) r=r>r?rQr#rrrArrrrrrtr&r&rdr'rs4  ArcseZdZfddZ         ddejdeejdeejd eejd eejd eeeejd ee d ee dee dee de eeje ffddZ Z S)AltRobertaEncoderc:t|_tfddtjD|_d|_dS)Ncg|]}tqSr&)rr6_rcr&r' z.AltRobertaEncoder.__init__..F) rPrQrcr! ModuleListrangenum_hidden_layerslayergradient_checkpointingrbrdrr'rQ   zAltRobertaEncoder.__init__NFTrrrrrpast_key_valuesrroutput_hidden_states return_dictrc  Cs^| rdnd} |r dnd} |r|jjrdnd} |jr%|jr%|r%tdd}|r)dnd}t|jD]^\}}| r;| |f} |durC||nd}|durM||nd}|jrc|jrc||j |||||||}n ||||||||}|d}|rz||df7}|r| |df} |jjr| |df} q0| r| |f} | st dd ||| | | fDSt ||| | | d S) Nr&zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FrrJrrycs|] }|dur|VqdSrr&r6vr&r&r'r:Esz,AltRobertaEncoder.forward..)last_hidden_staterr attentionscross_attentions) rcrrtraininglogger warning_once enumerater_gradient_checkpointing_func__call__r;r )r9rrrrrrrrrrZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheiZ layer_moduleZlayer_head_maskr layer_outputsr&r&r'rrsz         zAltRobertaEncoder.forward) NNNNNNFFT)r=r>r?rQr#rrrArrrr rrrtr&r&rdr'rsD      rcr)AltRobertaPoolercs*tt|j|j|_t|_dSr)rPrQr!rrSrZTanh activationrbrdr&r'rQ[s zAltRobertaPooler.__init__rrcCs(|dddf}||}||}|S)Nr)rr)r9rZfirst_token_tensor pooled_outputr&r&r'rr`s  zAltRobertaPooler.forwardrr&r&rdr'rZs rmodulerrrrscalingr\c Ks|t||dd|}|dur||}tjj|dtjd|j}tjj |||j d}t||} | dd } | |fS)NrJr)rrO)prrry) r#rrr!r"rZfloat32rrOr\rr) rrrrrrr\kwargs attn_weights attn_outputr&r&r'eager_attention_forwardjs  rcsheZdZdZfddZ   d dejdeejdeejd eed e ejeejff d d Z Z S)AltCLIPAttentionz=Multi-headed attention from 'Attention Is All You Need' papercst||_|j|_|j|_|j|j|_|j|j|jkr-td|jd|jd|jd|_ |j |_ d|_ t |j|j|_t |j|j|_t |j|j|_t |j|j|_dS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: ).F)rPrQrcrS embed_dimrz num_headshead_dimr{scaleZattention_dropoutr\ is_causalr!rk_projv_projq_projout_projrbrdr&r'rQs$   zAltCLIPAttention.__init__NFrrcausal_attention_maskrrc CsL|j\}}}||}||} ||} ||||j|jdd}| |||j|jdd} | |||j|jdd} |jj dkrY|durR|durR||}n |durX|}n|du|_ t } |jj dkrz|jj dkrt|rtt dnt|jj } | ||| | ||j |j|jsdn|jd \} } | |||} || } |sd} | | fS) z#Input shape: Batch x Time x ChannelrryZflash_attention_2NrZsdpaz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r)rrr\)rrrrrrrrrcrrrrrrrrr\reshaperr)r9rrrr batch_sizernrZqueriesr<valuesZattention_interfacerrr&r&r'rrsH          zAltCLIPAttention.forward)NNF) r=r>r?r@rQr#rrrrrrrtr&r&rdr'rs" rcr) AltCLIPMLPcsDt||_t|j|_t|j|j |_ t|j |j|_ dSr) rPrQrcr r activation_fnr!rrSrfc1fc2rbrdr&r'rQs  zAltCLIPMLP.__init__rrcCs"||}||}||}|Sr)r r r rr&r&r'rrs   zAltCLIPMLP.forwardrr&r&rdr'r s r c sTeZdZdeffdd Z d dejdejdejdeed e ej f d d Z Z S) AltCLIPEncoderLayerrccsRt|j|_t||_tj|j|jd|_ t ||_ tj|j|jd|_ dSr) rPrQrSrr self_attnr!rXrY layer_norm1r mlp layer_norm2rbrdr&r'rQs   zAltCLIPEncoderLayer.__init__FrrrrrcCsd|}||}|j||||d\}}||}|}||}||}||}|f}|r0||f7}|S)aI Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. `(config.encoder_attention_heads,)`. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. )rrrr)rrrr)r9rrrrZresidualrrr&r&r'rrs"     zAltCLIPEncoderLayer.forwardF) r=r>r?rrQr#rrrrrArrrtr&r&rdr'r s r csteZdZdZdeffdd Z     ddeejdeejdee d ee d ee d e e e ff d d Z ZS)AltCLIPEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`AltCLIPEncoderLayer`]. Args: config: AltCLIPConfig rccr)Ncrr&)r rrr&r'rrz+AltCLIPEncoder.__init__..F) rPrQrcr!rrrlayersrrbrdrr'rQrzAltCLIPEncoder.__init__Nrrrrrrc Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}|r"dnd}|r(dnd}|} t|jD]1\} } |r<|| f}|jrM|jrM|| j | |||} n| | |||d} | d} |rb|| df}q1|rj|| f}|sxt dd| ||fDSt | ||dS) a Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Causal mask for the text model. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nr&)rrrcsrrr&rr&r&r'r:lsz)AltCLIPEncoder.forward..)rrr) rcrruse_return_dictrrrrrrr;r ) r9rkrrrrrZencoder_statesZall_attentionsridxZ encoder_layerrr&r&r'rr sF&     zAltCLIPEncoder.forward)NNNNN)r=r>r?r@rrQrr#rrrrr rrrtr&r&rdr'rs*  rcsXeZdZdeffdd Zdejdededejfdd Zdd ej dejfd d Z Z S)AltCLIPVisionEmbeddingsrccst||_|j|_|j|_|j|_tt |j|_ tj |j |j|j|jdd|_|j|jd|_|jd|_t|j|j|_|jdt |jddddS)NF)Z in_channelsZ out_channelsZ kernel_sizeZstridebiasryrrHrIrK)rPrQrcrSr image_size patch_sizer! Parameterr#Zrandnclass_embeddingZConv2dZ num_channelspatch_embedding num_patches num_positionsrRposition_embeddingr]r$r^rbrdr&r'rQts"  "z AltCLIPVisionEmbeddings.__init__rqheightwidthrc Cs|jdd}|jjd}|jdd}tjs(||kr(||kr(||jS|ddddf}|ddddf}|jd} ||j} ||j} t |d} | d| | | }| dddd}t j j|| | fdd d }| dddddd| }tj||fdd S) a This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution images. This method is also adapted to support torch.jit tracing. Adapted from: - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211 rrNrJg?r ryZbicubicF)r`modeZ align_cornersr)rr weightrsr#Zjit is_tracingrHrrrrr!r"Z interpolaterr) r9rqr!r"rr rZclass_pos_embedZpatch_pos_embedrZ new_heightZ new_widthZsqrt_num_positionsr&r&r'interpolate_pos_encodings*      z0AltCLIPVisionEmbeddings.interpolate_pos_encodingF pixel_valuesc Cs|j\}}}}|s&||jks||jkr&td|d|d|jd|jd |jjj}||j|d}|ddd}|j |dd} t j | |gdd } |r[| | | ||} | S| ||j} | S) NzInput image size (*z) doesn't match model (rrNryrrJr)rrr{rr$rOrflattenrrr^r#rr&r rH) r9r'r&rrr!r"Z target_dtypeZ patch_embedsZ class_embedsrqr&r&r'rrs  zAltCLIPVisionEmbeddings.forwardr) r=r>r?rrQr#rr|r&rArrrtr&r&rdr'rss )rc@s$eZdZeZdZdZgZddZdS)AltCLIPPreTrainedModelZaltclipTcCs|jj}t|tr:|jj}tjj|jd|jd|dtjj|j j |jj |dtjj|j j |jj |ddSt|t r|jj}|jdd|jjd|}|jd|}tjj|jj |dtjj|jj |dtjj|jj |dtjj|jj |ddSt|tr|jj}|jjdd|jjd|}d|jjd|}tjj|jj |dtjj|jj |ddSt|trtjj|jj |jd|jjdd|j_tjj|jj |jd|jjdd|j_dSt|tjr|jj |j j!ddSt|tj"r%|j jjd|jjd|jdur#|jj dSdSt|tj#rH|j jjd|jjd|j$durJ|j j|j$ dSdSdS) zInitialize the weightsrr)meanstd)r,ryTg?N)%rcZinitializer_factorrrr!initZnormal_rrrr$Zinitializer_ranger rrrrrrr rSr r  AltCLIPModeltext_projectiontext_embed_dimZ_is_hf_initializedvisual_projectionvision_embed_dimrXrdataZzero_Zfill_rrRrD)r9rfactorZ in_proj_stdZ out_proj_stdZfc_stdr&r&r' _init_weightssZ          z$AltCLIPPreTrainedModel._init_weightsN) r=r>r?r config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_moduler5r&r&r&r'r*s  r*csreZdZdeffdd Ze     ddeejdee dee d ee d ee d e e e ff d d Z ZS)AltCLIPVisionTransformerrccsRt||_|j}t||_tj||jd|_ t ||_ tj||jd|_ dSr) rPrQrcrSrrqr!rXrY pre_layrnormrencoderpost_layernorm)r9rcrrdr&r'rQs   z!AltCLIPVisionTransformer.__init__NFr'rrrr&rc Cs|dur|n|jj}|dur|n|jj}|dur|n|jj}|dur&td|j||d}||}|j||||d}|d}|dddddf} || } |s[|| f|ddSt || |j |j dS)Nz You have to specify pixel_values)r&)rkrrrrrr pooler_outputrr) rcrrrr{rqr8r9r:r rr) r9r'rrrr&rencoder_outputsrrr&r&r'rrs2   z AltCLIPVisionTransformer.forward)NNNNF)r=r>r?rrQrrr#rArrrr rrrtr&r&rdr'r7s*  r7cseZdZeZdZdeffdd ZdejfddZ e    dde e j d e ed e ed ed e edeeeff ddZZS)AltCLIPVisionModelr'rccs"t|t||_|dSr)rPrQr7 vision_model post_initrbrdr&r'rQ5s   zAltCLIPVisionModel.__init__rcC |jjjSr)r?rqrr8r&r&r'get_input_embeddings; z'AltCLIPVisionModel.get_input_embeddingsNFrrr&rcCs(|dur|n|jj}|j|||||dS)a Examples: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, AltCLIPVisionModel >>> model = AltCLIPVisionModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled CLS states ```Nr'rrr&r)rcrr?)r9r'rrr&rr&r&r'rr>szAltCLIPVisionModel.forwardNNNFN)r=r>r?rr6Zmain_input_namerQr!ModulerBrrr#rArrrr rrrtr&r&rdr'r>1s0 r>a( The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in *Attention is all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 )Z custom_introc seZdZeZdfdd ZddZddZdd Ze dd e e j d e e j d e e j de e j de e j de e j de e j de e j de e e jde ede ede ede edeee j effddZZS)AltRobertaModelTcsDt|||_t||_t||_|rt|nd|_| dS)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer N) rPrQrcrCrqrr9rpoolerr@)r9rcadd_pooling_layerrdr&r'rQxs    zAltRobertaModel.__init__cCs|jjSrrqrTr8r&r&r'rBsz$AltRobertaModel.get_input_embeddingscCs ||j_dSrrJr9rr&r&r'set_input_embeddings z$AltRobertaModel.set_input_embeddingscCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsr9rrr)r9Zheads_to_prunerrr&r&r' _prune_headsszAltRobertaModel._prune_headsNrjrrMrHrrkrrrrrrrrc CsP| dur| n|jj} | dur| n|jj} | dur| n|jj} |jjr-| dur(| n|jj} nd} |dur;|dur;td|durJ||||}n|durW|dd}ntd|\}}|durf|j n|j }| durv| ddj dnd}|durt j |||f|d}|durt |jdr|jjddd|f}|||}|}n t j|t j|d }|||}|jjr|dur|\}}}||f}|durt j ||d}||}nd}|||jj}|j|||||d }|j|||||| | | | | d }|d}|jdur ||nd}| s||f|d dSt|||j|j|j|jd S)NFzDYou cannot specify both input_ids and inputs_embeds at the same timerJz5You have to specify either input_ids or inputs_embedsrryrrMrf)rjrHrMrkrl) rrrrrrrrrr)rr<rrrr)rcrrrrrr{Z%warn_if_padding_and_no_attention_maskr`r rr#ZonesrirqrMr^r_raZget_extended_attention_maskZinvert_attention_maskZ get_head_maskrr9rHrrrrr)r9rjrrMrHrrkrrrrrrrrmrrnr rlrorpZextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthrZencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputr=sequence_outputrr&r&r'rrs       zAltRobertaModel.forward)T) NNNNNNNNNNNNN)r=r>r?rr6rQrBrLrOrrr#rrrArrrrrrrtr&r&rdr'rGfsb     rGcseZdZeZfddZdejfddZdej ddfdd Z dd e e dej ffd d Z e           dd e ejde ejde ejde ejde ejde ejde ejde ejde ede ede edeeeffddZZS)AltCLIPTextModelcsLt|t|dd|_t|j|j|_tj |j|j d|_ | dS)NF)rIrE) rPrQrGrobertar!rrS project_dimtransformationrXrYpre_LNr@rbrdr&r'rQ s  zAltCLIPTextModel.__init__rcCrArrRrqrTr8r&r&r'rBrCz%AltCLIPTextModel.get_input_embeddingsrNcCs||jj_dSrrVrKr&r&r'rLsz%AltCLIPTextModel.set_input_embeddingsnew_num_tokenscs t|Sr)rPresize_token_embeddings)r9rWrdr&r'rXrMz(AltCLIPTextModel.resize_token_embeddingsrjrrMrHrrkrrrrrc  Cs| dur| n|jj} |j||||||||| | | d } | d} || } || }|dddf}| s<||f| ddSt||| j| jdS)a+ Examples: ```python >>> from transformers import AutoProcessor, AltCLIPTextModel >>> model = AltCLIPTextModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> texts = ["it's a cat", "it's a dog"] >>> inputs = processor(text=texts, padding=True, return_tensors="pt") >>> outputs = model(**inputs) >>> last_hidden_state = outputs.last_hidden_state >>> pooled_output = outputs.pooler_output # pooled CLS states ```N) rjrrMrHrrkrrrrrrryr;)rcrrRrUrTrrr)r9rjrrMrHrrkrrrrrrrPZprojection_stater<r&r&r'rrs4!  zAltCLIPTextModel.forwardr) NNNNNNNNNNN)r=r>r?rr6rQr!rFrBrRrLrr|rXrr#rrrrrrrrtr&r&rdr'rQsV      rQcsLeZdZeZdeffdd Ze       ddeej deej deej dee d ee d ee d ej fd d Z e     ddeej dee d ee de d ee d ej f ddZ e          ddeejdeej deej deejdeej dee dee d ee de d ee d eeeffddZZS)r.rccst|t|jtstdt|jdt|jts(tdt|jd|j}|j}|j |_ |j |_ |j |_ t||_t||_tj|j |j dd|_tj|j |j dd|_tt|jj|_|dS)NzRconfig.vision_config is expected to be of type AltCLIPVisionConfig but is of type .zNconfig.text_config is expected to be of type AltCLIPTextConfig but is of type F)r)rPrQr vision_configr TypeErrortype text_configrZprojection_dimrSr0rSr2rQ text_modelr7r?r!rr1r/rr#rrcZlogit_scale_init_value logit_scaler@)r9rcr^r[rdr&r'rQcs0      zAltCLIPModel.__init__NrjrrHrrrrc Csj|dur|n|jj}|dur|n|jj}|dur|n|jj}|j|||||||d}|d} || } | S)a Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPTextModel`]. Examples: ```python >>> from transformers import AutoProcessor, AltCLIPModel >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> inputs = processor(text=["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) ```N)rjrrHrMrrrr)rcrrrr_r/) r9rjrrHrMrrr text_outputsrZ text_featuresr&r&r'get_text_featuress  zAltCLIPModel.get_text_featuresFr'r&c Csf|dur|n|jj}|dur|n|jj}|dur|n|jj}|j|||||d}|d}||}|S)a* Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPVisionModel`]. Examples: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, AltCLIPModel >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> image_features = model.get_image_features(**inputs) ```NrDr)rcrrrr?r1) r9r'rrr&rvision_outputsrZimage_featuresr&r&r'get_image_featuress zAltCLIPModel.get_image_featuresrM return_lossc  Cs(|dur|n|jj}|dur|n|jj}| dur| n|jj} |j||||||| d} |j|||| | d} | d} || } | d}||}| | jdddd} ||jdddd}|j }t || |}|j}d}|rtt|}| s|||| | | f}|dur|f|S|St||||| | | d S) a return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. Examples: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, AltCLIPModel >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP") >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor( ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True ... ) >>> outputs = model(**inputs) >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities ```N)rjrrMrHrrrrDrryrJT)rrZkeepdim)r-r.r/r0r1r2r3)rcrrrr_r?r1r/Znormr`expr#rr*Tr+r,)r9rjr'rrHrMrerrr&rrarcr1r0r`r/r.r-rr&r&r'rrsX%    zAltCLIPModel.forward)NNNNNNNrE) NNNNNNNNFN)r=r>r?rr6rQrrr#rrrArbrdZ LongTensorrrr,rrrtr&r&rdr'r.`s . /    r.cCs6||}tj|dd|||}||S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rr)ner|r#ZcumsumZtype_asra)rjrDrlmaskZincremental_indicesr&r&r'rgAs  rg)r*r>rQr.)r)r)Hr@r dataclassesrtypingrrrrrrr#Ztorch.nnr!Ztorch.utils.checkpointZ activationsr Zmodeling_outputsr r r rrZmodeling_utilsrrZ pytorch_utilsrrrutilsrrrrZconfiguration_altcliprrrZ get_loggerr=rrr(r+r,rFrCrurrrrrrrrfloatrrr r rrr*r7r>rGrQr.rg__all__r&r&r&r's     %Z4W^ P2bS465Y b